You are viewing a plain text version of this content. The canonical link for it is here.
Posted to reviews@spark.apache.org by yanboliang <gi...@git.apache.org> on 2016/02/09 16:42:49 UTC
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
GitHub user yanboliang opened a pull request:
https://github.com/apache/spark/pull/11136
[SPARK-12811] [ML] Estimator for Generalized Linear Models(GLMs)
Estimator for Generalized Linear Models(GLMs)
You can merge this pull request into a Git repository by running:
$ git pull https://github.com/yanboliang/spark spark-12811
Alternatively you can review and apply these changes as the patch at:
https://github.com/apache/spark/pull/11136.patch
To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:
This closes #11136
----
commit 5af604e180d92a75b18b30717d6516d5ee8135bd
Author: Yanbo Liang <yb...@gmail.com>
Date: 2016-02-09T15:37:27Z
Initial version of Generalized Linear Regression
----
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r53722484
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,547 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ if (isDefined(link)) {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
+ s"Generalized Linear Regression with ${$(family)} family does not support ${$(link)} " +
+ s"link function.")
+ }
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor and
+ * a description of the error distribution.
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("genLinReg"))
+
+ /**
+ * Set the name of family which is a description of the error distribution
+ * to be used in the model.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+
+ /**
+ * Set the name of the model link function.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Set if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+ setDefault(fitIntercept -> true)
+
+ /**
+ * Set the maximum number of iterations.
+ * Default is 100.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+ setDefault(maxIter -> 100)
+
+ /**
+ * Set the convergence tolerance of iterations.
+ * Smaller value will lead to higher accuracy with the cost of more iterations.
+ * Default is 1E-6.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setTol(value: Double): this.type = set(tol, value)
+ setDefault(tol -> 1E-6)
+
+ /**
+ * Set the regularization parameter.
+ * Default is 0.0.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setRegParam(value: Double): this.type = set(regParam, value)
+ setDefault(regParam -> 0.0)
+
+ /**
+ * Whether to over-/under-sample training instances according to the given weights in weightCol.
+ * If empty, all instances are treated equally (weight 1.0).
+ * Default is empty, so all instances have weight one.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setWeightCol(value: String): this.type = set(weightCol, value)
+ setDefault(weightCol -> "")
+
+ /**
+ * Set the solver algorithm used for optimization.
+ * Currently only support "irls" which is also the default solver.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setSolver(value: String): this.type = set(solver, value)
+ setDefault(solver -> "irls")
+
+ override protected def train(dataset: DataFrame): GeneralizedLinearRegressionModel = {
+ val familyLink = $(family) match {
+ case "gaussian" => if (isDefined(link)) Gaussian($(link)) else Gaussian("identity")
--- End diff --
If we have the canonical link defined as the default value, we can use `Gaussian()` here instead of `Gaussian("identity")`, just to avoid string references in the implementation. Also see my comments below about handling raw strings in `Family` and `Link`.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r53722773
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,547 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
--- End diff --
We can call it `GeneralizedLinearRegressionBase` because it also implements other functions.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by AmplabJenkins <gi...@git.apache.org>.
Github user AmplabJenkins commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-189211002
Merged build finished. Test PASSed.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by AmplabJenkins <gi...@git.apache.org>.
Github user AmplabJenkins commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-182897395
Test FAILed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/51097/
Test FAILed.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r53722678
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,547 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ if (isDefined(link)) {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
+ s"Generalized Linear Regression with ${$(family)} family does not support ${$(link)} " +
+ s"link function.")
+ }
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor and
+ * a description of the error distribution.
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("genLinReg"))
+
+ /**
+ * Set the name of family which is a description of the error distribution
+ * to be used in the model.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+
+ /**
+ * Set the name of the model link function.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Set if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+ setDefault(fitIntercept -> true)
+
+ /**
+ * Set the maximum number of iterations.
+ * Default is 100.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+ setDefault(maxIter -> 100)
+
+ /**
+ * Set the convergence tolerance of iterations.
+ * Smaller value will lead to higher accuracy with the cost of more iterations.
+ * Default is 1E-6.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setTol(value: Double): this.type = set(tol, value)
+ setDefault(tol -> 1E-6)
+
+ /**
+ * Set the regularization parameter.
+ * Default is 0.0.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setRegParam(value: Double): this.type = set(regParam, value)
+ setDefault(regParam -> 0.0)
+
+ /**
+ * Whether to over-/under-sample training instances according to the given weights in weightCol.
+ * If empty, all instances are treated equally (weight 1.0).
+ * Default is empty, so all instances have weight one.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setWeightCol(value: String): this.type = set(weightCol, value)
+ setDefault(weightCol -> "")
+
+ /**
+ * Set the solver algorithm used for optimization.
+ * Currently only support "irls" which is also the default solver.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setSolver(value: String): this.type = set(solver, value)
+ setDefault(solver -> "irls")
+
+ override protected def train(dataset: DataFrame): GeneralizedLinearRegressionModel = {
+ val familyLink = $(family) match {
+ case "gaussian" => if (isDefined(link)) Gaussian($(link)) else Gaussian("identity")
+ case "binomial" => if (isDefined(link)) Binomial($(link)) else Binomial("logit")
+ case "poisson" => if (isDefined(link)) Poisson($(link)) else Poisson("log")
+ case "gamma" => if (isDefined(link)) Gamma($(link)) else Gamma("inverse")
+ }
+
+ val numFeatures = dataset.select(col($(featuresCol))).limit(1).map {
+ case Row(features: Vector) => features.size
+ }.first()
+ if (numFeatures > 4096) {
+ val msg = "Currently, GeneralizedLinearRegression only supports number of features" +
+ s" <= 4096. Found $numFeatures in the input dataset."
+ throw new SparkException(msg)
+ }
+
+ val w = if ($(weightCol).isEmpty) lit(1.0) else col($(weightCol))
+ val instances: RDD[Instance] = dataset.select(
+ col($(labelCol)), w, col($(featuresCol))).map {
+ case Row(label: Double, weight: Double, features: Vector) =>
+ Instance(label, weight, features)
+ }
+
+ if ($(family) == "gaussian" && $(link) == "identity") {
+ val optimizer = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true)
+ val wlsModel = optimizer.fit(instances)
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ wlsModel.coefficients, wlsModel.intercept).setParent(this))
+ return model
+ }
+
+ val newInstances = instances.map { instance =>
+ val mu = familyLink.initialize(instance.label, instance.weight)
+ val eta = familyLink.predict(mu)
+ Instance(eta, instance.weight, instance.features)
+ }
+
+ val initialModel = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true).fit(newInstances)
+
+ val reweightFunc: (Instance, WeightedLeastSquaresModel) => (Double, Double) = {
+ (instance: Instance, model: WeightedLeastSquaresModel) => {
+ val eta = model.predict(instance.features)
+ val mu = familyLink.fitted(eta)
+ val z = familyLink.adjusted(instance.label, mu, eta)
+ val w = familyLink.weights(mu) * instance.weight
+ (z, w)
+ }
+ }
+
+ val optimizer = new IterativelyReweightedLeastSquares(initialModel, reweightFunc,
+ $(fitIntercept), $(regParam), $(maxIter), $(tol))
+
+ val irlsModel = optimizer.fit(instances)
+
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ irlsModel.coefficients, irlsModel.intercept).setParent(this))
+ model
+ }
+
+ @Since("2.0.0")
+ override def copy(extra: ParamMap): GeneralizedLinearRegression = defaultCopy(extra)
+}
+
+@Since("2.0.0")
+object GeneralizedLinearRegression {
+
+ /** Set of families that GeneralizedLinearRegression supports */
+ private[ml] val supportedFamilies = Set("gaussian", "binomial", "poisson", "gamma")
+
+ /** Set of links that GeneralizedLinearRegression supports */
+ private[ml] val supportedLinks = Set("identity", "log", "inverse", "logit", "probit",
+ "cloglog", "sqrt")
+
+ /** Set of family and link pairs that GeneralizedLinearRegression supports */
+ private[ml] val supportedFamilyLinkPairs = Set(
+ "gaussian" -> "identity", "gaussian" -> "log", "gaussian" -> "inverse",
+ "binomial" -> "logit", "binomial" -> "probit", "binomial" -> "cloglog",
+ "poisson" -> "log", "poisson" -> "identity", "poisson" -> "sqrt",
+ "gamma" -> "inverse", "gamma" -> "identity", "gamma" -> "log"
+ )
+}
+
+/**
+ * :: Experimental ::
+ * Model produced by [[GeneralizedLinearRegression]].
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegressionModel private[ml] (
+ @Since("2.0.0") override val uid: String,
+ @Since("2.0.0") val coefficients: Vector,
+ @Since("2.0.0") val intercept: Double)
+ extends RegressionModel[Vector, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams {
+
+ private lazy val familyLink = $(family) match {
+ case "gaussian" => if (isDefined(link)) Gaussian($(link)) else Gaussian("identity")
+ case "binomial" => if (isDefined(link)) Binomial($(link)) else Binomial("logit")
+ case "poisson" => if (isDefined(link)) Poisson($(link)) else Poisson("log")
+ case "gamma" => if (isDefined(link)) Gamma($(link)) else Gamma("inverse")
+ }
--- End diff --
This is duplicated code. We can factor this out to `GLMBase`, `Family`, and `Link`.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r53722443
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,547 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ if (isDefined(link)) {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
+ s"Generalized Linear Regression with ${$(family)} family does not support ${$(link)} " +
+ s"link function.")
+ }
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor and
+ * a description of the error distribution.
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("genLinReg"))
+
+ /**
+ * Set the name of family which is a description of the error distribution
+ * to be used in the model.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+
+ /**
+ * Set the name of the model link function.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Set if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+ setDefault(fitIntercept -> true)
+
+ /**
+ * Set the maximum number of iterations.
+ * Default is 100.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+ setDefault(maxIter -> 100)
+
+ /**
+ * Set the convergence tolerance of iterations.
+ * Smaller value will lead to higher accuracy with the cost of more iterations.
+ * Default is 1E-6.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setTol(value: Double): this.type = set(tol, value)
+ setDefault(tol -> 1E-6)
+
+ /**
+ * Set the regularization parameter.
+ * Default is 0.0.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setRegParam(value: Double): this.type = set(regParam, value)
+ setDefault(regParam -> 0.0)
+
+ /**
+ * Whether to over-/under-sample training instances according to the given weights in weightCol.
--- End diff --
This is not accurate. Weighted samples are not necessarily sampled. We should just follow the doc from `HasWeightCol` or say `Sets [[weightCol]].`.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r53722503
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,547 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ if (isDefined(link)) {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
+ s"Generalized Linear Regression with ${$(family)} family does not support ${$(link)} " +
+ s"link function.")
+ }
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor and
+ * a description of the error distribution.
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("genLinReg"))
+
+ /**
+ * Set the name of family which is a description of the error distribution
+ * to be used in the model.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+
+ /**
+ * Set the name of the model link function.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Set if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+ setDefault(fitIntercept -> true)
+
+ /**
+ * Set the maximum number of iterations.
+ * Default is 100.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+ setDefault(maxIter -> 100)
+
+ /**
+ * Set the convergence tolerance of iterations.
+ * Smaller value will lead to higher accuracy with the cost of more iterations.
+ * Default is 1E-6.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setTol(value: Double): this.type = set(tol, value)
+ setDefault(tol -> 1E-6)
+
+ /**
+ * Set the regularization parameter.
+ * Default is 0.0.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setRegParam(value: Double): this.type = set(regParam, value)
+ setDefault(regParam -> 0.0)
+
+ /**
+ * Whether to over-/under-sample training instances according to the given weights in weightCol.
+ * If empty, all instances are treated equally (weight 1.0).
+ * Default is empty, so all instances have weight one.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setWeightCol(value: String): this.type = set(weightCol, value)
+ setDefault(weightCol -> "")
+
+ /**
+ * Set the solver algorithm used for optimization.
+ * Currently only support "irls" which is also the default solver.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setSolver(value: String): this.type = set(solver, value)
+ setDefault(solver -> "irls")
+
+ override protected def train(dataset: DataFrame): GeneralizedLinearRegressionModel = {
+ val familyLink = $(family) match {
+ case "gaussian" => if (isDefined(link)) Gaussian($(link)) else Gaussian("identity")
+ case "binomial" => if (isDefined(link)) Binomial($(link)) else Binomial("logit")
+ case "poisson" => if (isDefined(link)) Poisson($(link)) else Poisson("log")
+ case "gamma" => if (isDefined(link)) Gamma($(link)) else Gamma("inverse")
+ }
+
+ val numFeatures = dataset.select(col($(featuresCol))).limit(1).map {
+ case Row(features: Vector) => features.size
+ }.first()
+ if (numFeatures > 4096) {
--- End diff --
Shall we add a constant `WeightedLeastSquares.MaxNumFeatures` and call it here?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r53722641
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,547 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ if (isDefined(link)) {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
+ s"Generalized Linear Regression with ${$(family)} family does not support ${$(link)} " +
+ s"link function.")
+ }
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor and
+ * a description of the error distribution.
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("genLinReg"))
+
+ /**
+ * Set the name of family which is a description of the error distribution
+ * to be used in the model.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+
+ /**
+ * Set the name of the model link function.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Set if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+ setDefault(fitIntercept -> true)
+
+ /**
+ * Set the maximum number of iterations.
+ * Default is 100.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+ setDefault(maxIter -> 100)
+
+ /**
+ * Set the convergence tolerance of iterations.
+ * Smaller value will lead to higher accuracy with the cost of more iterations.
+ * Default is 1E-6.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setTol(value: Double): this.type = set(tol, value)
+ setDefault(tol -> 1E-6)
+
+ /**
+ * Set the regularization parameter.
+ * Default is 0.0.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setRegParam(value: Double): this.type = set(regParam, value)
+ setDefault(regParam -> 0.0)
+
+ /**
+ * Whether to over-/under-sample training instances according to the given weights in weightCol.
+ * If empty, all instances are treated equally (weight 1.0).
+ * Default is empty, so all instances have weight one.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setWeightCol(value: String): this.type = set(weightCol, value)
+ setDefault(weightCol -> "")
+
+ /**
+ * Set the solver algorithm used for optimization.
+ * Currently only support "irls" which is also the default solver.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setSolver(value: String): this.type = set(solver, value)
+ setDefault(solver -> "irls")
+
+ override protected def train(dataset: DataFrame): GeneralizedLinearRegressionModel = {
+ val familyLink = $(family) match {
+ case "gaussian" => if (isDefined(link)) Gaussian($(link)) else Gaussian("identity")
+ case "binomial" => if (isDefined(link)) Binomial($(link)) else Binomial("logit")
+ case "poisson" => if (isDefined(link)) Poisson($(link)) else Poisson("log")
+ case "gamma" => if (isDefined(link)) Gamma($(link)) else Gamma("inverse")
+ }
+
+ val numFeatures = dataset.select(col($(featuresCol))).limit(1).map {
+ case Row(features: Vector) => features.size
+ }.first()
+ if (numFeatures > 4096) {
+ val msg = "Currently, GeneralizedLinearRegression only supports number of features" +
+ s" <= 4096. Found $numFeatures in the input dataset."
+ throw new SparkException(msg)
+ }
+
+ val w = if ($(weightCol).isEmpty) lit(1.0) else col($(weightCol))
+ val instances: RDD[Instance] = dataset.select(
+ col($(labelCol)), w, col($(featuresCol))).map {
+ case Row(label: Double, weight: Double, features: Vector) =>
+ Instance(label, weight, features)
+ }
+
+ if ($(family) == "gaussian" && $(link) == "identity") {
+ val optimizer = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true)
+ val wlsModel = optimizer.fit(instances)
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ wlsModel.coefficients, wlsModel.intercept).setParent(this))
+ return model
+ }
+
+ val newInstances = instances.map { instance =>
+ val mu = familyLink.initialize(instance.label, instance.weight)
+ val eta = familyLink.predict(mu)
+ Instance(eta, instance.weight, instance.features)
+ }
+
+ val initialModel = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true).fit(newInstances)
+
+ val reweightFunc: (Instance, WeightedLeastSquaresModel) => (Double, Double) = {
+ (instance: Instance, model: WeightedLeastSquaresModel) => {
+ val eta = model.predict(instance.features)
+ val mu = familyLink.fitted(eta)
+ val z = familyLink.adjusted(instance.label, mu, eta)
+ val w = familyLink.weights(mu) * instance.weight
+ (z, w)
+ }
+ }
+
+ val optimizer = new IterativelyReweightedLeastSquares(initialModel, reweightFunc,
+ $(fitIntercept), $(regParam), $(maxIter), $(tol))
+
+ val irlsModel = optimizer.fit(instances)
+
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ irlsModel.coefficients, irlsModel.intercept).setParent(this))
+ model
+ }
+
+ @Since("2.0.0")
+ override def copy(extra: ParamMap): GeneralizedLinearRegression = defaultCopy(extra)
+}
+
+@Since("2.0.0")
+object GeneralizedLinearRegression {
+
+ /** Set of families that GeneralizedLinearRegression supports */
+ private[ml] val supportedFamilies = Set("gaussian", "binomial", "poisson", "gamma")
--- End diff --
String references are error-prone. I would first define:
~~~scala
lazy val supportedFamilyAndLinkParis = Set(Gaussian(Identity), Gaussian(Log), ...)
~~~
and then
~~~scala
lazy val supportedFamilyNames = supportedFamilies.map(_.name)
lazy val supportedLinkNames = supportedFamilies.map(_.link.name)
~~~
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-181945930
**[Test build #50980 has started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/50980/consoleFull)** for PR 11136 at commit [`3082686`](https://github.com/apache/spark/commit/3082686d61a851c240f16a9cd7aafc97fa6e3296).
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by dbtsai <gi...@git.apache.org>.
Github user dbtsai commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r54141380
--- Diff: mllib/src/main/scala/org/apache/spark/ml/optim/WeightedLeastSquares.scala ---
@@ -157,6 +157,12 @@ private[ml] class WeightedLeastSquares(
private[ml] object WeightedLeastSquares {
/**
+ * In order to take the normal equation approach efficiently, [[WeightedLeastSquares]]
+ * only supports the number of features is no more than 4096.
+ */
+ val MaxNumFeatures: Int = 4096
--- End diff --
For constant, do we have naming convention? Like `MAX_NUM_FEATURES`?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by AmplabJenkins <gi...@git.apache.org>.
Github user AmplabJenkins commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-181949577
Test PASSed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/50978/
Test PASSed.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by yanboliang <gi...@git.apache.org>.
Github user yanboliang commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r53909458
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,565 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionBase extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * Default is "gaussian".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "The name of family which is a description of the error distribution to be used in the " +
+ "model. Supported options: gaussian(default), binomial, poisson and gamma.",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilyNames.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of link function which provides the relationship
+ * between the linear predictor and the mean of the distribution function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "The name of link function " +
+ "which provides the relationship between the linear predictor and the mean of the " +
+ "distribution function. Supported options: identity, log, inverse, logit, probit, " +
+ "cloglog and sqrt.",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinkNames.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ import GeneralizedLinearRegression._
+ protected lazy val familyObj = Family.fromName($(family))
+ protected lazy val linkObj = if (isDefined(link)) {
+ Link.fromName($(link))
+ } else {
+ familyObj.defaultLink
+ }
+ protected lazy val familyAndLink = new FamilyAndLink(familyObj, linkObj)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ if ($(solver) == "irls") {
+ setDefault(maxIter -> 25)
+ }
+ if (isDefined(link)) {
+ require(GeneralizedLinearRegression.supportedFamilyAndLinkParis.contains(
+ familyObj -> linkObj), s"Generalized Linear Regression with ${$(family)} family " +
+ s"does not support ${$(link)} link function.")
+ }
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor (link function) and
+ * a description of the error distribution (family).
+ * It supports "gaussian", "binomial", "poisson" and "gamma" as family.
+ * Valid link functions for each family is listed below. The first link function of each family
+ * is the default one.
+ * - "gaussian" -> "identity", "log", "inverse"
+ * - "binomial" -> "logit", "probit", "cloglog"
+ * - "poisson" -> "log", "identity", "sqrt"
+ * - "gamma" -> "inverse", "identity", "log"
--- End diff --
Mention the list of valid (family, link) combinations at here.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by dbtsai <gi...@git.apache.org>.
Github user dbtsai commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-189192045
Gonna do another detail pass of the code tomorrow.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r54059562
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,565 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionBase extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * Default is "gaussian".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "The name of family which is a description of the error distribution to be used in the " +
+ "model. Supported options: gaussian(default), binomial, poisson and gamma.",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilyNames.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of link function which provides the relationship
+ * between the linear predictor and the mean of the distribution function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "The name of link function " +
+ "which provides the relationship between the linear predictor and the mean of the " +
+ "distribution function. Supported options: identity, log, inverse, logit, probit, " +
+ "cloglog and sqrt.",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinkNames.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ import GeneralizedLinearRegression._
+ protected lazy val familyObj = Family.fromName($(family))
+ protected lazy val linkObj = if (isDefined(link)) {
+ Link.fromName($(link))
+ } else {
+ familyObj.defaultLink
+ }
+ protected lazy val familyAndLink = new FamilyAndLink(familyObj, linkObj)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ if ($(solver) == "irls") {
+ setDefault(maxIter -> 25)
+ }
+ if (isDefined(link)) {
+ require(GeneralizedLinearRegression.supportedFamilyAndLinkParis.contains(
+ familyObj -> linkObj), s"Generalized Linear Regression with ${$(family)} family " +
+ s"does not support ${$(link)} link function.")
+ }
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor (link function) and
+ * a description of the error distribution (family).
+ * It supports "gaussian", "binomial", "poisson" and "gamma" as family.
+ * Valid link functions for each family is listed below. The first link function of each family
+ * is the default one.
+ * - "gaussian" -> "identity", "log", "inverse"
+ * - "binomial" -> "logit", "probit", "cloglog"
+ * - "poisson" -> "log", "identity", "sqrt"
+ * - "gamma" -> "inverse", "identity", "log"
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionBase with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("glm"))
+
+ /**
+ * Sets the value of param [[family]].
+ * Default is "gaussian".
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+ setDefault(family -> "gaussian")
+
+ /**
+ * Sets the value of param [[link]].
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Sets if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+
+ /**
+ * Sets the maximum number of iterations.
+ * Default is 25 if the solver algorithm is "irls".
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+
+ /**
+ * Sets the convergence tolerance of iterations.
+ * Smaller value will lead to higher accuracy with the cost of more iterations.
+ * Default is 1E-6.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setTol(value: Double): this.type = set(tol, value)
+ setDefault(tol -> 1E-6)
+
+ /**
+ * Sets the regularization parameter.
+ * Default is 0.0.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setRegParam(value: Double): this.type = set(regParam, value)
+ setDefault(regParam -> 0.0)
+
+ /**
+ * Sets the value of param [[weightCol]].
+ * If this is not set or empty, we treat all instance weights as 1.0.
+ * Default is empty, so all instances have weight one.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setWeightCol(value: String): this.type = set(weightCol, value)
+ setDefault(weightCol -> "")
+
+ /**
+ * Sets the solver algorithm used for optimization.
+ * Currently only support "irls" which is also the default solver.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setSolver(value: String): this.type = set(solver, value)
+ setDefault(solver -> "irls")
+
+ override protected def train(dataset: DataFrame): GeneralizedLinearRegressionModel = {
+ val numFeatures = dataset.select(col($(featuresCol))).limit(1)
+ .map { case Row(features: Vector) =>
+ features.size
+ }.first()
+ if (numFeatures > WeightedLeastSquares.MaxNumFeatures) {
+ val msg = "Currently, GeneralizedLinearRegression only supports number of features" +
+ s" <= ${WeightedLeastSquares.MaxNumFeatures}. Found $numFeatures in the input dataset."
+ throw new SparkException(msg)
+ }
+
+ val w = if ($(weightCol).isEmpty) lit(1.0) else col($(weightCol))
+ val instances: RDD[Instance] = dataset.select(col($(labelCol)), w, col($(featuresCol)))
+ .map { case Row(label: Double, weight: Double, features: Vector) =>
+ Instance(label, weight, features)
+ }
+
+ if (familyObj == GeneralizedLinearRegression.Gaussian &&
--- End diff --
We can `import GeneralizedLinearRegression._` in `class GeneralizedLinearRegression` to save some code.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by AmplabJenkins <gi...@git.apache.org>.
Github user AmplabJenkins commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-182935191
Test FAILed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/51101/
Test FAILed.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by AmplabJenkins <gi...@git.apache.org>.
Github user AmplabJenkins commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-190527136
Test FAILed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/52214/
Test FAILed.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r53722629
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,547 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ if (isDefined(link)) {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
+ s"Generalized Linear Regression with ${$(family)} family does not support ${$(link)} " +
+ s"link function.")
+ }
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor and
+ * a description of the error distribution.
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("genLinReg"))
+
+ /**
+ * Set the name of family which is a description of the error distribution
+ * to be used in the model.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+
+ /**
+ * Set the name of the model link function.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Set if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+ setDefault(fitIntercept -> true)
+
+ /**
+ * Set the maximum number of iterations.
+ * Default is 100.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+ setDefault(maxIter -> 100)
+
+ /**
+ * Set the convergence tolerance of iterations.
+ * Smaller value will lead to higher accuracy with the cost of more iterations.
+ * Default is 1E-6.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setTol(value: Double): this.type = set(tol, value)
+ setDefault(tol -> 1E-6)
+
+ /**
+ * Set the regularization parameter.
+ * Default is 0.0.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setRegParam(value: Double): this.type = set(regParam, value)
+ setDefault(regParam -> 0.0)
+
+ /**
+ * Whether to over-/under-sample training instances according to the given weights in weightCol.
+ * If empty, all instances are treated equally (weight 1.0).
+ * Default is empty, so all instances have weight one.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setWeightCol(value: String): this.type = set(weightCol, value)
+ setDefault(weightCol -> "")
+
+ /**
+ * Set the solver algorithm used for optimization.
+ * Currently only support "irls" which is also the default solver.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setSolver(value: String): this.type = set(solver, value)
+ setDefault(solver -> "irls")
+
+ override protected def train(dataset: DataFrame): GeneralizedLinearRegressionModel = {
+ val familyLink = $(family) match {
+ case "gaussian" => if (isDefined(link)) Gaussian($(link)) else Gaussian("identity")
+ case "binomial" => if (isDefined(link)) Binomial($(link)) else Binomial("logit")
+ case "poisson" => if (isDefined(link)) Poisson($(link)) else Poisson("log")
+ case "gamma" => if (isDefined(link)) Gamma($(link)) else Gamma("inverse")
+ }
+
+ val numFeatures = dataset.select(col($(featuresCol))).limit(1).map {
+ case Row(features: Vector) => features.size
+ }.first()
+ if (numFeatures > 4096) {
+ val msg = "Currently, GeneralizedLinearRegression only supports number of features" +
+ s" <= 4096. Found $numFeatures in the input dataset."
+ throw new SparkException(msg)
+ }
+
+ val w = if ($(weightCol).isEmpty) lit(1.0) else col($(weightCol))
+ val instances: RDD[Instance] = dataset.select(
+ col($(labelCol)), w, col($(featuresCol))).map {
+ case Row(label: Double, weight: Double, features: Vector) =>
+ Instance(label, weight, features)
+ }
+
+ if ($(family) == "gaussian" && $(link) == "identity") {
+ val optimizer = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true)
+ val wlsModel = optimizer.fit(instances)
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ wlsModel.coefficients, wlsModel.intercept).setParent(this))
+ return model
+ }
+
+ val newInstances = instances.map { instance =>
+ val mu = familyLink.initialize(instance.label, instance.weight)
+ val eta = familyLink.predict(mu)
+ Instance(eta, instance.weight, instance.features)
+ }
+
+ val initialModel = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true).fit(newInstances)
--- End diff --
ditto: better alignment
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r53722686
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,547 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ if (isDefined(link)) {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
+ s"Generalized Linear Regression with ${$(family)} family does not support ${$(link)} " +
+ s"link function.")
+ }
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor and
+ * a description of the error distribution.
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("genLinReg"))
+
+ /**
+ * Set the name of family which is a description of the error distribution
+ * to be used in the model.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+
+ /**
+ * Set the name of the model link function.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Set if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+ setDefault(fitIntercept -> true)
+
+ /**
+ * Set the maximum number of iterations.
+ * Default is 100.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+ setDefault(maxIter -> 100)
+
+ /**
+ * Set the convergence tolerance of iterations.
+ * Smaller value will lead to higher accuracy with the cost of more iterations.
+ * Default is 1E-6.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setTol(value: Double): this.type = set(tol, value)
+ setDefault(tol -> 1E-6)
+
+ /**
+ * Set the regularization parameter.
+ * Default is 0.0.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setRegParam(value: Double): this.type = set(regParam, value)
+ setDefault(regParam -> 0.0)
+
+ /**
+ * Whether to over-/under-sample training instances according to the given weights in weightCol.
+ * If empty, all instances are treated equally (weight 1.0).
+ * Default is empty, so all instances have weight one.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setWeightCol(value: String): this.type = set(weightCol, value)
+ setDefault(weightCol -> "")
+
+ /**
+ * Set the solver algorithm used for optimization.
+ * Currently only support "irls" which is also the default solver.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setSolver(value: String): this.type = set(solver, value)
+ setDefault(solver -> "irls")
+
+ override protected def train(dataset: DataFrame): GeneralizedLinearRegressionModel = {
+ val familyLink = $(family) match {
+ case "gaussian" => if (isDefined(link)) Gaussian($(link)) else Gaussian("identity")
+ case "binomial" => if (isDefined(link)) Binomial($(link)) else Binomial("logit")
+ case "poisson" => if (isDefined(link)) Poisson($(link)) else Poisson("log")
+ case "gamma" => if (isDefined(link)) Gamma($(link)) else Gamma("inverse")
+ }
+
+ val numFeatures = dataset.select(col($(featuresCol))).limit(1).map {
+ case Row(features: Vector) => features.size
+ }.first()
+ if (numFeatures > 4096) {
+ val msg = "Currently, GeneralizedLinearRegression only supports number of features" +
+ s" <= 4096. Found $numFeatures in the input dataset."
+ throw new SparkException(msg)
+ }
+
+ val w = if ($(weightCol).isEmpty) lit(1.0) else col($(weightCol))
+ val instances: RDD[Instance] = dataset.select(
+ col($(labelCol)), w, col($(featuresCol))).map {
+ case Row(label: Double, weight: Double, features: Vector) =>
+ Instance(label, weight, features)
+ }
+
+ if ($(family) == "gaussian" && $(link) == "identity") {
+ val optimizer = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true)
+ val wlsModel = optimizer.fit(instances)
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ wlsModel.coefficients, wlsModel.intercept).setParent(this))
+ return model
+ }
+
+ val newInstances = instances.map { instance =>
+ val mu = familyLink.initialize(instance.label, instance.weight)
+ val eta = familyLink.predict(mu)
+ Instance(eta, instance.weight, instance.features)
+ }
+
+ val initialModel = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true).fit(newInstances)
+
+ val reweightFunc: (Instance, WeightedLeastSquaresModel) => (Double, Double) = {
+ (instance: Instance, model: WeightedLeastSquaresModel) => {
+ val eta = model.predict(instance.features)
+ val mu = familyLink.fitted(eta)
+ val z = familyLink.adjusted(instance.label, mu, eta)
+ val w = familyLink.weights(mu) * instance.weight
+ (z, w)
+ }
+ }
+
+ val optimizer = new IterativelyReweightedLeastSquares(initialModel, reweightFunc,
+ $(fitIntercept), $(regParam), $(maxIter), $(tol))
+
+ val irlsModel = optimizer.fit(instances)
+
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ irlsModel.coefficients, irlsModel.intercept).setParent(this))
+ model
+ }
+
+ @Since("2.0.0")
+ override def copy(extra: ParamMap): GeneralizedLinearRegression = defaultCopy(extra)
+}
+
+@Since("2.0.0")
+object GeneralizedLinearRegression {
+
+ /** Set of families that GeneralizedLinearRegression supports */
+ private[ml] val supportedFamilies = Set("gaussian", "binomial", "poisson", "gamma")
+
+ /** Set of links that GeneralizedLinearRegression supports */
+ private[ml] val supportedLinks = Set("identity", "log", "inverse", "logit", "probit",
+ "cloglog", "sqrt")
+
+ /** Set of family and link pairs that GeneralizedLinearRegression supports */
+ private[ml] val supportedFamilyLinkPairs = Set(
+ "gaussian" -> "identity", "gaussian" -> "log", "gaussian" -> "inverse",
+ "binomial" -> "logit", "binomial" -> "probit", "binomial" -> "cloglog",
+ "poisson" -> "log", "poisson" -> "identity", "poisson" -> "sqrt",
+ "gamma" -> "inverse", "gamma" -> "identity", "gamma" -> "log"
+ )
+}
+
+/**
+ * :: Experimental ::
+ * Model produced by [[GeneralizedLinearRegression]].
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegressionModel private[ml] (
+ @Since("2.0.0") override val uid: String,
+ @Since("2.0.0") val coefficients: Vector,
+ @Since("2.0.0") val intercept: Double)
+ extends RegressionModel[Vector, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams {
+
+ private lazy val familyLink = $(family) match {
+ case "gaussian" => if (isDefined(link)) Gaussian($(link)) else Gaussian("identity")
+ case "binomial" => if (isDefined(link)) Binomial($(link)) else Binomial("logit")
+ case "poisson" => if (isDefined(link)) Poisson($(link)) else Poisson("log")
+ case "gamma" => if (isDefined(link)) Gamma($(link)) else Gamma("inverse")
+ }
+
+ override protected def predict(features: Vector): Double = {
+ val eta = BLAS.dot(features, coefficients) + intercept
+ familyLink.fitted(eta)
+ }
+
+ @Since("2.0.0")
+ override def copy(extra: ParamMap): GeneralizedLinearRegressionModel = {
+ copyValues(new GeneralizedLinearRegressionModel(uid, coefficients, intercept), extra)
+ .setParent(parent)
+ }
+}
+
+/**
+ * A description of the error distribution and link function to be used in the model.
+ * @param link a link function instance
+ */
+private[ml] abstract class Family(val link: Link) extends Serializable {
--- End diff --
I suggest moving this class inside `object GeneralizedLinearRegression`. Otherwise, it lives under `ml.regression`.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-182942965
**[Test build #51109 has started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/51109/consoleFull)** for PR 11136 at commit [`97c3f6a`](https://github.com/apache/spark/commit/97c3f6a2b94d6705606dd020f61c5c604b562b4c).
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r54508381
--- Diff: mllib/src/test/scala/org/apache/spark/ml/regression/GeneralizedLinearRegressionSuite.scala ---
@@ -0,0 +1,499 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import scala.util.Random
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.ml.param.ParamsSuite
+import org.apache.spark.ml.util.MLTestingUtils
+import org.apache.spark.mllib.classification.LogisticRegressionSuite._
+import org.apache.spark.mllib.linalg.{BLAS, DenseVector, Vectors}
+import org.apache.spark.mllib.random._
+import org.apache.spark.mllib.regression.LabeledPoint
+import org.apache.spark.mllib.util.MLlibTestSparkContext
+import org.apache.spark.mllib.util.TestingUtils._
+import org.apache.spark.sql.{DataFrame, Row}
+
+class GeneralizedLinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
+
+ private val seed: Int = 42
+ @transient var datasetGaussianIdentity: DataFrame = _
+ @transient var datasetGaussianLog: DataFrame = _
+ @transient var datasetGaussianInverse: DataFrame = _
+ @transient var datasetBinomial: DataFrame = _
+ @transient var datasetPoissonLog: DataFrame = _
+ @transient var datasetPoissonIdentity: DataFrame = _
+ @transient var datasetPoissonSqrt: DataFrame = _
+ @transient var datasetGammaInverse: DataFrame = _
+ @transient var datasetGammaIdentity: DataFrame = _
+ @transient var datasetGammaLog: DataFrame = _
+
+ override def beforeAll(): Unit = {
+ super.beforeAll()
+
+ import GeneralizedLinearRegressionSuite._
+
+ datasetGaussianIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "identity"), 2))
+
+ datasetGaussianLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "log"), 2))
+
+ datasetGaussianInverse = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "inverse"), 2))
+
+ datasetBinomial = {
+ val nPoints = 10000
+ val coefficients = Array(-0.57997, 0.912083, -0.371077, -0.819866, 2.688191)
+ val xMean = Array(5.843, 3.057, 3.758, 1.199)
+ val xVariance = Array(0.6856, 0.1899, 3.116, 0.581)
+
+ val testData =
+ generateMultinomialLogisticInput(coefficients, xMean, xVariance, true, nPoints, seed)
+
+ sqlContext.createDataFrame(sc.parallelize(testData, 4))
+ }
+
+ datasetPoissonLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "log"), 2))
+
+ datasetPoissonIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "identity"), 2))
+
+ datasetPoissonSqrt = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "sqrt"), 2))
+
+ datasetGammaInverse = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "inverse"), 2))
+
+ datasetGammaIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "identity"), 2))
+
+ datasetGammaLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "log"), 2))
+ }
+
+ test("params") {
+ ParamsSuite.checkParams(new GeneralizedLinearRegression)
+ val model = new GeneralizedLinearRegressionModel("genLinReg", Vectors.dense(0.0), 0.0)
+ ParamsSuite.checkParams(model)
+ }
+
+ test("generalized linear regression: default params") {
+ val glr = new GeneralizedLinearRegression
+ assert(glr.getLabelCol === "label")
+ assert(glr.getFeaturesCol === "features")
+ assert(glr.getPredictionCol === "prediction")
+ assert(glr.getFitIntercept)
+ assert(glr.getTol === 1E-6)
+ assert(glr.getWeightCol === "")
+ assert(glr.getRegParam === 0.0)
+ assert(glr.getSolver == "irls")
+ val model = glr.setFamily("gaussian").setLink("identity")
--- End diff --
Leave a TODO here. We should construct the model in directly in unit testing instead of via fitting.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r54508365
--- Diff: mllib/src/test/scala/org/apache/spark/ml/regression/GeneralizedLinearRegressionSuite.scala ---
@@ -0,0 +1,499 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import scala.util.Random
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.ml.param.ParamsSuite
+import org.apache.spark.ml.util.MLTestingUtils
+import org.apache.spark.mllib.classification.LogisticRegressionSuite._
+import org.apache.spark.mllib.linalg.{BLAS, DenseVector, Vectors}
+import org.apache.spark.mllib.random._
+import org.apache.spark.mllib.regression.LabeledPoint
+import org.apache.spark.mllib.util.MLlibTestSparkContext
+import org.apache.spark.mllib.util.TestingUtils._
+import org.apache.spark.sql.{DataFrame, Row}
+
+class GeneralizedLinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
+
+ private val seed: Int = 42
+ @transient var datasetGaussianIdentity: DataFrame = _
+ @transient var datasetGaussianLog: DataFrame = _
+ @transient var datasetGaussianInverse: DataFrame = _
+ @transient var datasetBinomial: DataFrame = _
+ @transient var datasetPoissonLog: DataFrame = _
+ @transient var datasetPoissonIdentity: DataFrame = _
+ @transient var datasetPoissonSqrt: DataFrame = _
+ @transient var datasetGammaInverse: DataFrame = _
+ @transient var datasetGammaIdentity: DataFrame = _
+ @transient var datasetGammaLog: DataFrame = _
+
+ override def beforeAll(): Unit = {
+ super.beforeAll()
+
+ import GeneralizedLinearRegressionSuite._
+
+ datasetGaussianIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "identity"), 2))
+
+ datasetGaussianLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "log"), 2))
+
+ datasetGaussianInverse = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "inverse"), 2))
+
+ datasetBinomial = {
+ val nPoints = 10000
+ val coefficients = Array(-0.57997, 0.912083, -0.371077, -0.819866, 2.688191)
+ val xMean = Array(5.843, 3.057, 3.758, 1.199)
+ val xVariance = Array(0.6856, 0.1899, 3.116, 0.581)
+
+ val testData =
+ generateMultinomialLogisticInput(coefficients, xMean, xVariance, true, nPoints, seed)
--- End diff --
it would be good to say `addIntercept = true` instead of just `true`.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by AmplabJenkins <gi...@git.apache.org>.
Github user AmplabJenkins commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-190522465
Test FAILed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/52211/
Test FAILed.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r53722637
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,547 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ if (isDefined(link)) {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
+ s"Generalized Linear Regression with ${$(family)} family does not support ${$(link)} " +
+ s"link function.")
+ }
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor and
+ * a description of the error distribution.
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("genLinReg"))
+
+ /**
+ * Set the name of family which is a description of the error distribution
+ * to be used in the model.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+
+ /**
+ * Set the name of the model link function.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Set if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+ setDefault(fitIntercept -> true)
+
+ /**
+ * Set the maximum number of iterations.
+ * Default is 100.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+ setDefault(maxIter -> 100)
+
+ /**
+ * Set the convergence tolerance of iterations.
+ * Smaller value will lead to higher accuracy with the cost of more iterations.
+ * Default is 1E-6.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setTol(value: Double): this.type = set(tol, value)
+ setDefault(tol -> 1E-6)
+
+ /**
+ * Set the regularization parameter.
+ * Default is 0.0.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setRegParam(value: Double): this.type = set(regParam, value)
+ setDefault(regParam -> 0.0)
+
+ /**
+ * Whether to over-/under-sample training instances according to the given weights in weightCol.
+ * If empty, all instances are treated equally (weight 1.0).
+ * Default is empty, so all instances have weight one.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setWeightCol(value: String): this.type = set(weightCol, value)
+ setDefault(weightCol -> "")
+
+ /**
+ * Set the solver algorithm used for optimization.
+ * Currently only support "irls" which is also the default solver.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setSolver(value: String): this.type = set(solver, value)
+ setDefault(solver -> "irls")
+
+ override protected def train(dataset: DataFrame): GeneralizedLinearRegressionModel = {
+ val familyLink = $(family) match {
+ case "gaussian" => if (isDefined(link)) Gaussian($(link)) else Gaussian("identity")
+ case "binomial" => if (isDefined(link)) Binomial($(link)) else Binomial("logit")
+ case "poisson" => if (isDefined(link)) Poisson($(link)) else Poisson("log")
+ case "gamma" => if (isDefined(link)) Gamma($(link)) else Gamma("inverse")
+ }
+
+ val numFeatures = dataset.select(col($(featuresCol))).limit(1).map {
+ case Row(features: Vector) => features.size
+ }.first()
+ if (numFeatures > 4096) {
+ val msg = "Currently, GeneralizedLinearRegression only supports number of features" +
+ s" <= 4096. Found $numFeatures in the input dataset."
+ throw new SparkException(msg)
+ }
+
+ val w = if ($(weightCol).isEmpty) lit(1.0) else col($(weightCol))
+ val instances: RDD[Instance] = dataset.select(
+ col($(labelCol)), w, col($(featuresCol))).map {
+ case Row(label: Double, weight: Double, features: Vector) =>
+ Instance(label, weight, features)
+ }
+
+ if ($(family) == "gaussian" && $(link) == "identity") {
+ val optimizer = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true)
+ val wlsModel = optimizer.fit(instances)
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ wlsModel.coefficients, wlsModel.intercept).setParent(this))
+ return model
+ }
+
+ val newInstances = instances.map { instance =>
+ val mu = familyLink.initialize(instance.label, instance.weight)
+ val eta = familyLink.predict(mu)
+ Instance(eta, instance.weight, instance.features)
+ }
+
+ val initialModel = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true).fit(newInstances)
+
+ val reweightFunc: (Instance, WeightedLeastSquaresModel) => (Double, Double) = {
+ (instance: Instance, model: WeightedLeastSquaresModel) => {
+ val eta = model.predict(instance.features)
+ val mu = familyLink.fitted(eta)
+ val z = familyLink.adjusted(instance.label, mu, eta)
+ val w = familyLink.weights(mu) * instance.weight
+ (z, w)
+ }
+ }
+
+ val optimizer = new IterativelyReweightedLeastSquares(initialModel, reweightFunc,
+ $(fitIntercept), $(regParam), $(maxIter), $(tol))
+
+ val irlsModel = optimizer.fit(instances)
+
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ irlsModel.coefficients, irlsModel.intercept).setParent(this))
+ model
+ }
+
+ @Since("2.0.0")
+ override def copy(extra: ParamMap): GeneralizedLinearRegression = defaultCopy(extra)
+}
+
+@Since("2.0.0")
+object GeneralizedLinearRegression {
--- End diff --
The object should be `private[ml]` too.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by yanboliang <gi...@git.apache.org>.
Github user yanboliang commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r52610253
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,472 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.Logging
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
+ s"Generalized Linear Regression with ${$(family)} family does not support ${$(link)} " +
+ s"link function.")
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor and
+ * a description of the error distribution.
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("genLinReg"))
+
+ /**
+ * Set the name of family which is a description of the error distribution
+ * to be used in the model.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+
+ /**
+ * Set the name of the model link function.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Set if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+ setDefault(fitIntercept -> true)
+
+ /**
+ * Set the maximum number of iterations.
+ * Default is 100.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+ setDefault(maxIter -> 100)
+
+ /**
+ * Set the convergence tolerance of iterations.
+ * Smaller value will lead to higher accuracy with the cost of more iterations.
+ * Default is 1E-6.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setTol(value: Double): this.type = set(tol, value)
+ setDefault(tol -> 1E-6)
+
+ /**
+ * Set the regularization parameter.
+ * Default is 0.0.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setRegParam(value: Double): this.type = set(regParam, value)
+ setDefault(regParam -> 0.0)
+
+ /**
+ * Whether to over-/under-sample training instances according to the given weights in weightCol.
+ * If empty, all instances are treated equally (weight 1.0).
+ * Default is empty, so all instances have weight one.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setWeightCol(value: String): this.type = set(weightCol, value)
+ setDefault(weightCol -> "")
+
+ /**
+ * Set the solver algorithm used for optimization.
+ * Currently only support "irls" which is also the default solver.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setSolver(value: String): this.type = set(solver, value)
+ setDefault(solver -> "irls")
+
+ override protected def train(dataset: DataFrame): GeneralizedLinearRegressionModel = {
+ val familyLink = $(family) match {
+ case "gaussian" => Gaussian($(link))
+ case "binomial" => Binomial($(link))
+ case "poisson" => Poisson($(link))
+ case "gamma" => Gamma($(link))
+ }
+
+ val w = if ($(weightCol).isEmpty) lit(1.0) else col($(weightCol))
+ val instances: RDD[Instance] = dataset.select(
+ col($(labelCol)), w, col($(featuresCol))).map {
+ case Row(label: Double, weight: Double, features: Vector) =>
+ Instance(label, weight, features)
+ }
+
+ if ($(family) == "gaussian" && $(link) == "identity") {
+ val optimizer = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true)
+ val wlsModel = optimizer.fit(instances)
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ wlsModel.coefficients, wlsModel.intercept).setParent(this))
+ return model
+ }
+
+ val newInstances = instances.map { instance =>
+ val mu = familyLink.initialize(instance.label, instance.weight)
+ val eta = familyLink.predict(mu)
+ Instance(eta, instance.weight, instance.features)
+ }
+
+ val initialModel = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true).fit(newInstances)
+
+ val reweightFunc: (Instance, WeightedLeastSquaresModel) => (Double, Double) = {
+ (instance: Instance, model: WeightedLeastSquaresModel) => {
+ val eta = model.predict(instance.features)
+ val mu = familyLink.fitted(eta)
+ val z = familyLink.adjusted(instance.label, mu, eta)
+ val w = familyLink.weights(mu) * instance.weight
+ (z, w)
+ }
+ }
+
+ val optimizer = new IterativelyReweightedLeastSquares(initialModel, reweightFunc,
+ $(fitIntercept), $(regParam), $(maxIter), $(tol))
+
+ val irlsModel = optimizer.fit(instances)
+
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ irlsModel.coefficients, irlsModel.intercept).setParent(this))
+ model
+ }
+
+ @Since("2.0.0")
+ override def copy(extra: ParamMap): GeneralizedLinearRegression = defaultCopy(extra)
+}
+
+@Since("2.0.0")
+object GeneralizedLinearRegression {
+
+ /** Set of families that GeneralizedLinearRegression supports */
+ private[ml] val supportedFamilies = Set("gaussian", "binomial", "poisson", "gamma")
+
+ /** Set of links that GeneralizedLinearRegression supports */
+ private[ml] val supportedLinks = Set("identity", "log", "inverse", "logit", "probit",
+ "cloglog", "sqrt")
+
+ /** Set of family and link pairs that GeneralizedLinearRegression supports */
+ private[ml] val supportedFamilyLinkPairs = Set(
+ "gaussian" -> "identity", "gaussian" -> "log", "gaussian" -> "inverse",
+ "binomial" -> "logit", "binomial" -> "probit", "binomial" -> "cloglog",
+ "poisson" -> "log", "poisson" -> "identity", "poisson" -> "sqrt",
+ "gamma" -> "inverse", "gamma" -> "identity", "gamma" -> "log"
+ )
+}
+
+/**
+ * :: Experimental ::
+ * Model produced by [[GeneralizedLinearRegression]].
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegressionModel private[ml] (
+ @Since("2.0.0") override val uid: String,
+ @Since("2.0.0") val coefficients: Vector,
+ @Since("2.0.0") val intercept: Double)
+ extends RegressionModel[Vector, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams {
+
+ private lazy val familyLink = $(family) match {
+ case "gaussian" => Gaussian($(link))
+ case "binomial" => Binomial($(link))
+ case "poisson" => Poisson($(link))
+ case "gamma" => Gamma($(link))
+ }
+
+ override protected def predict(features: Vector): Double = {
+ val eta = BLAS.dot(features, coefficients) + intercept
+ familyLink.fitted(eta)
+ }
+
+ @Since("2.0.0")
+ override def copy(extra: ParamMap): GeneralizedLinearRegressionModel = {
+ copyValues(new GeneralizedLinearRegressionModel(uid, coefficients, intercept), extra)
+ .setParent(parent)
+ }
+}
+
+/**
+ * A description of the error distribution and link function to be used in the model.
+ * @param link a link function instance
+ */
+private[ml] abstract class Family(val link: Link) extends Serializable {
+
+ /** Initialize the starting value for mu. */
+ def initialize(y: Double, weight: Double): Double
+
+ /** The variance of mu to its mean. */
+ def variance(mu: Double): Double
+
+ /** Weights for IRLS steps. */
+ def weights(mu: Double): Double = {
+ 1.0 / (math.pow(this.link.deriv(mu), 2.0) * this.variance(mu))
+ }
+
+ /** The adjusted response variable. */
+ def adjusted(y: Double, mu: Double, eta: Double): Double = {
+ eta + (y - mu) * link.deriv(mu)
+ }
+
+ /** Linear predictors based on given mu. */
+ def predict(mu: Double): Double = this.link.link(mu)
+
+ /** Fitted values based on linear predictors eta. */
+ def fitted(eta: Double): Double = this.link.unlink(eta)
+}
+
+/**
+ * Gaussian exponential family distribution.
+ * The default link for the Gaussian family is the identity link.
+ * @param link a link function instance
+ */
+private[ml] class Gaussian(link: Link = Identity) extends Family(link) {
+
+ override def initialize(y: Double, weight: Double): Double = y
+
+ def variance(mu: Double): Double = 1.0
+}
+
+private[ml] object Gaussian {
+
+ def apply(link: String): Gaussian = {
+ link match {
+ case "identity" => new Gaussian(Identity)
+ case "log" => new Gaussian(Log)
+ case "inverse" => new Gaussian(Inverse)
+ }
+ }
+}
+
+/**
+ * Binomial exponential family distribution.
+ * The default link for the Binomial family is the logit link.
+ * @param link a link function instance
+ */
+private[ml] class Binomial(link: Link = Logit) extends Family(link) {
+
+ override def initialize(y: Double, weight: Double): Double = {
+ (weight * y + 0.5) / (weight + 1.0)
+ }
+
+ override def variance(mu: Double): Double = mu * (1 - mu)
+}
+
+private[ml] object Binomial {
+
+ def apply(link: String): Binomial = {
+ link match {
+ case "logit" => new Binomial(Logit)
+ case "probit" => new Binomial(Probit)
+ case "cloglog" => new Binomial(CLogLog)
+ }
+ }
+}
+
+/**
+ * Poisson exponential family distribution.
+ * The default link for the Poisson family is the log link.
+ * @param link a link function instance
+ */
+private[ml] class Poisson(link: Link = Log) extends Family(link) {
+
+ override def initialize(y: Double, weight: Double): Double = y + 0.1
+
+ override def variance(mu: Double): Double = mu
+}
+
+private[ml] object Poisson {
+
+ def apply(link: String): Poisson = {
+ link match {
+ case "log" => new Poisson(Log)
+ case "sqrt" => new Poisson(Sqrt)
+ case "identity" => new Poisson(Identity)
+ }
+ }
+}
+
+/**
+ * Gamma exponential family distribution.
+ * The default link for the Gamma family is the inverse link.
+ * @param link a link function instance
+ */
+private[ml] class Gamma(link: Link = Log) extends Family(link) {
+
+ override def initialize(y: Double, weight: Double): Double = y
+
+ override def variance(mu: Double): Double = math.pow(mu, 2.0)
+}
+
+private[ml] object Gamma {
+
+ def apply(link: String): Gamma = {
+ link match {
+ case "inverse" => new Gamma(Inverse)
+ case "identity" => new Gamma(Identity)
+ case "log" => new Gamma(Log)
+ }
+ }
+}
+
+/**
+ * A description of the link function to be used in the model.
+ */
+private[ml] trait Link extends Serializable {
+
+ /** The link function. */
+ def link(mu: Double): Double
+
+ /** The derivative function. */
+ def deriv(mu: Double): Double
+
+ /** The inverse link function. */
+ def unlink(eta: Double): Double
+}
+
+private[ml] object Identity extends Link {
+
+ override def link(mu: Double): Double = mu
+
+ override def deriv(mu: Double): Double = 1.0
+
+ override def unlink(eta: Double): Double = eta
+}
+
+private[ml] object Logit extends Link {
+
+ override def link(mu: Double): Double = math.log(mu / (1.0 - mu))
+
+ override def deriv(mu: Double): Double = 1.0 / (mu * (1.0 - mu))
+
+ override def unlink(eta: Double): Double = 1.0 / (1.0 + math.exp(-1.0 * eta))
+}
--- End diff --
I think we should have restriction both at ```Family``` and ```Link```. For ```Family``` we can limit the input ```y``` is valid such as in the range ```[0,1]``` for ```Binomial``` family. For ```Link``` we should ensure ```math.log(1.0 - x)``` not produce ```Infinity```.
I'm trying to use some function like ```numpy.clip``` to solve this issue, but it's not satisfied. For example:
Scala```
def clean(x: Double): Double = {
if (x < Double.MinPositiveValue) {
Double.MinPositiveValue
} else if (x > 1.0 - Double.MinPositiveValue) {
1.0 - Double.MinPositiveValue
} else {
x
}
}
```
The output is:
```
scala> clean(0.0)
res0: Double = 4.9E-324
scala> clean(1.0)
res1: Double = 1.0
```
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r53722694
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,547 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ if (isDefined(link)) {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
+ s"Generalized Linear Regression with ${$(family)} family does not support ${$(link)} " +
+ s"link function.")
+ }
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor and
+ * a description of the error distribution.
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("genLinReg"))
+
+ /**
+ * Set the name of family which is a description of the error distribution
+ * to be used in the model.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+
+ /**
+ * Set the name of the model link function.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Set if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+ setDefault(fitIntercept -> true)
+
+ /**
+ * Set the maximum number of iterations.
+ * Default is 100.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+ setDefault(maxIter -> 100)
+
+ /**
+ * Set the convergence tolerance of iterations.
+ * Smaller value will lead to higher accuracy with the cost of more iterations.
+ * Default is 1E-6.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setTol(value: Double): this.type = set(tol, value)
+ setDefault(tol -> 1E-6)
+
+ /**
+ * Set the regularization parameter.
+ * Default is 0.0.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setRegParam(value: Double): this.type = set(regParam, value)
+ setDefault(regParam -> 0.0)
+
+ /**
+ * Whether to over-/under-sample training instances according to the given weights in weightCol.
+ * If empty, all instances are treated equally (weight 1.0).
+ * Default is empty, so all instances have weight one.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setWeightCol(value: String): this.type = set(weightCol, value)
+ setDefault(weightCol -> "")
+
+ /**
+ * Set the solver algorithm used for optimization.
+ * Currently only support "irls" which is also the default solver.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setSolver(value: String): this.type = set(solver, value)
+ setDefault(solver -> "irls")
+
+ override protected def train(dataset: DataFrame): GeneralizedLinearRegressionModel = {
+ val familyLink = $(family) match {
+ case "gaussian" => if (isDefined(link)) Gaussian($(link)) else Gaussian("identity")
+ case "binomial" => if (isDefined(link)) Binomial($(link)) else Binomial("logit")
+ case "poisson" => if (isDefined(link)) Poisson($(link)) else Poisson("log")
+ case "gamma" => if (isDefined(link)) Gamma($(link)) else Gamma("inverse")
+ }
+
+ val numFeatures = dataset.select(col($(featuresCol))).limit(1).map {
+ case Row(features: Vector) => features.size
+ }.first()
+ if (numFeatures > 4096) {
+ val msg = "Currently, GeneralizedLinearRegression only supports number of features" +
+ s" <= 4096. Found $numFeatures in the input dataset."
+ throw new SparkException(msg)
+ }
+
+ val w = if ($(weightCol).isEmpty) lit(1.0) else col($(weightCol))
+ val instances: RDD[Instance] = dataset.select(
+ col($(labelCol)), w, col($(featuresCol))).map {
+ case Row(label: Double, weight: Double, features: Vector) =>
+ Instance(label, weight, features)
+ }
+
+ if ($(family) == "gaussian" && $(link) == "identity") {
+ val optimizer = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true)
+ val wlsModel = optimizer.fit(instances)
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ wlsModel.coefficients, wlsModel.intercept).setParent(this))
+ return model
+ }
+
+ val newInstances = instances.map { instance =>
+ val mu = familyLink.initialize(instance.label, instance.weight)
+ val eta = familyLink.predict(mu)
+ Instance(eta, instance.weight, instance.features)
+ }
+
+ val initialModel = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true).fit(newInstances)
+
+ val reweightFunc: (Instance, WeightedLeastSquaresModel) => (Double, Double) = {
+ (instance: Instance, model: WeightedLeastSquaresModel) => {
+ val eta = model.predict(instance.features)
+ val mu = familyLink.fitted(eta)
+ val z = familyLink.adjusted(instance.label, mu, eta)
+ val w = familyLink.weights(mu) * instance.weight
+ (z, w)
+ }
+ }
+
+ val optimizer = new IterativelyReweightedLeastSquares(initialModel, reweightFunc,
+ $(fitIntercept), $(regParam), $(maxIter), $(tol))
+
+ val irlsModel = optimizer.fit(instances)
+
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ irlsModel.coefficients, irlsModel.intercept).setParent(this))
+ model
+ }
+
+ @Since("2.0.0")
+ override def copy(extra: ParamMap): GeneralizedLinearRegression = defaultCopy(extra)
+}
+
+@Since("2.0.0")
+object GeneralizedLinearRegression {
+
+ /** Set of families that GeneralizedLinearRegression supports */
+ private[ml] val supportedFamilies = Set("gaussian", "binomial", "poisson", "gamma")
+
+ /** Set of links that GeneralizedLinearRegression supports */
+ private[ml] val supportedLinks = Set("identity", "log", "inverse", "logit", "probit",
+ "cloglog", "sqrt")
+
+ /** Set of family and link pairs that GeneralizedLinearRegression supports */
+ private[ml] val supportedFamilyLinkPairs = Set(
+ "gaussian" -> "identity", "gaussian" -> "log", "gaussian" -> "inverse",
+ "binomial" -> "logit", "binomial" -> "probit", "binomial" -> "cloglog",
+ "poisson" -> "log", "poisson" -> "identity", "poisson" -> "sqrt",
+ "gamma" -> "inverse", "gamma" -> "identity", "gamma" -> "log"
+ )
+}
+
+/**
+ * :: Experimental ::
+ * Model produced by [[GeneralizedLinearRegression]].
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegressionModel private[ml] (
+ @Since("2.0.0") override val uid: String,
+ @Since("2.0.0") val coefficients: Vector,
+ @Since("2.0.0") val intercept: Double)
+ extends RegressionModel[Vector, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams {
+
+ private lazy val familyLink = $(family) match {
+ case "gaussian" => if (isDefined(link)) Gaussian($(link)) else Gaussian("identity")
+ case "binomial" => if (isDefined(link)) Binomial($(link)) else Binomial("logit")
+ case "poisson" => if (isDefined(link)) Poisson($(link)) else Poisson("log")
+ case "gamma" => if (isDefined(link)) Gamma($(link)) else Gamma("inverse")
+ }
+
+ override protected def predict(features: Vector): Double = {
+ val eta = BLAS.dot(features, coefficients) + intercept
+ familyLink.fitted(eta)
+ }
+
+ @Since("2.0.0")
+ override def copy(extra: ParamMap): GeneralizedLinearRegressionModel = {
+ copyValues(new GeneralizedLinearRegressionModel(uid, coefficients, intercept), extra)
+ .setParent(parent)
+ }
+}
+
+/**
+ * A description of the error distribution and link function to be used in the model.
+ * @param link a link function instance
+ */
+private[ml] abstract class Family(val link: Link) extends Serializable {
+
+ /** Initialize the starting value for mu. */
+ def initialize(y: Double, weight: Double): Double
+
+ /** The variance of the endogenous variable's mean, given the value mu. */
+ def variance(mu: Double): Double
+
+ /** Weights for IRLS steps. */
+ def weights(mu: Double): Double = {
+ val x = clean(mu)
+ 1.0 / (math.pow(this.link.deriv(x), 2.0) * this.variance(x))
+ }
+
+ /** The adjusted response variable. */
+ def adjusted(y: Double, mu: Double, eta: Double): Double = {
+ val x = clean(mu)
+ eta + (y - x) * link.deriv(x)
+ }
+
+ /** Linear predictor based on given mu. */
+ def predict(mu: Double): Double = this.link.link(clean(mu))
+
+ /** Fitted value based on linear predictor eta. */
+ def fitted(eta: Double): Double = clean(this.link.unlink(eta))
+
+ /** Trim the fitted value so that it will be in valid range. */
+ def clean(mu: Double): Double = mu
--- End diff --
Does R use `clean` as the name? `project` seems a better name here.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-190527127
**[Test build #52214 has finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/52214/consoleFull)** for PR 11136 at commit [`314b562`](https://github.com/apache/spark/commit/314b562f315723a7117851289c8f5b6e1b16a6ac).
* This patch **fails to build**.
* This patch merges cleanly.
* This patch adds no public classes.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by AmplabJenkins <gi...@git.apache.org>.
Github user AmplabJenkins commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-190607484
Merged build finished. Test PASSed.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r53722437
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,547 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ if (isDefined(link)) {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
+ s"Generalized Linear Regression with ${$(family)} family does not support ${$(link)} " +
+ s"link function.")
+ }
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor and
+ * a description of the error distribution.
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("genLinReg"))
+
+ /**
+ * Set the name of family which is a description of the error distribution
+ * to be used in the model.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+
+ /**
+ * Set the name of the model link function.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Set if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+ setDefault(fitIntercept -> true)
+
+ /**
+ * Set the maximum number of iterations.
+ * Default is 100.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+ setDefault(maxIter -> 100)
--- End diff --
What is the default value in R? Btw, we might need to assign different default values for IRLS and L-BFGS.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-190526388
**[Test build #52214 has started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/52214/consoleFull)** for PR 11136 at commit [`314b562`](https://github.com/apache/spark/commit/314b562f315723a7117851289c8f5b6e1b16a6ac).
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r54059585
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,565 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionBase extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * Default is "gaussian".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "The name of family which is a description of the error distribution to be used in the " +
+ "model. Supported options: gaussian(default), binomial, poisson and gamma.",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilyNames.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of link function which provides the relationship
+ * between the linear predictor and the mean of the distribution function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "The name of link function " +
+ "which provides the relationship between the linear predictor and the mean of the " +
+ "distribution function. Supported options: identity, log, inverse, logit, probit, " +
+ "cloglog and sqrt.",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinkNames.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ import GeneralizedLinearRegression._
+ protected lazy val familyObj = Family.fromName($(family))
+ protected lazy val linkObj = if (isDefined(link)) {
+ Link.fromName($(link))
+ } else {
+ familyObj.defaultLink
+ }
+ protected lazy val familyAndLink = new FamilyAndLink(familyObj, linkObj)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ if ($(solver) == "irls") {
+ setDefault(maxIter -> 25)
+ }
+ if (isDefined(link)) {
+ require(GeneralizedLinearRegression.supportedFamilyAndLinkParis.contains(
+ familyObj -> linkObj), s"Generalized Linear Regression with ${$(family)} family " +
+ s"does not support ${$(link)} link function.")
+ }
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor (link function) and
+ * a description of the error distribution (family).
+ * It supports "gaussian", "binomial", "poisson" and "gamma" as family.
+ * Valid link functions for each family is listed below. The first link function of each family
+ * is the default one.
+ * - "gaussian" -> "identity", "log", "inverse"
+ * - "binomial" -> "logit", "probit", "cloglog"
+ * - "poisson" -> "log", "identity", "sqrt"
+ * - "gamma" -> "inverse", "identity", "log"
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionBase with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("glm"))
+
+ /**
+ * Sets the value of param [[family]].
+ * Default is "gaussian".
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+ setDefault(family -> "gaussian")
+
+ /**
+ * Sets the value of param [[link]].
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Sets if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+
+ /**
+ * Sets the maximum number of iterations.
+ * Default is 25 if the solver algorithm is "irls".
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+
+ /**
+ * Sets the convergence tolerance of iterations.
+ * Smaller value will lead to higher accuracy with the cost of more iterations.
+ * Default is 1E-6.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setTol(value: Double): this.type = set(tol, value)
+ setDefault(tol -> 1E-6)
+
+ /**
+ * Sets the regularization parameter.
+ * Default is 0.0.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setRegParam(value: Double): this.type = set(regParam, value)
+ setDefault(regParam -> 0.0)
+
+ /**
+ * Sets the value of param [[weightCol]].
+ * If this is not set or empty, we treat all instance weights as 1.0.
+ * Default is empty, so all instances have weight one.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setWeightCol(value: String): this.type = set(weightCol, value)
+ setDefault(weightCol -> "")
+
+ /**
+ * Sets the solver algorithm used for optimization.
+ * Currently only support "irls" which is also the default solver.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setSolver(value: String): this.type = set(solver, value)
+ setDefault(solver -> "irls")
+
+ override protected def train(dataset: DataFrame): GeneralizedLinearRegressionModel = {
+ val numFeatures = dataset.select(col($(featuresCol))).limit(1)
+ .map { case Row(features: Vector) =>
+ features.size
+ }.first()
+ if (numFeatures > WeightedLeastSquares.MaxNumFeatures) {
+ val msg = "Currently, GeneralizedLinearRegression only supports number of features" +
+ s" <= ${WeightedLeastSquares.MaxNumFeatures}. Found $numFeatures in the input dataset."
+ throw new SparkException(msg)
+ }
+
+ val w = if ($(weightCol).isEmpty) lit(1.0) else col($(weightCol))
+ val instances: RDD[Instance] = dataset.select(col($(labelCol)), w, col($(featuresCol)))
+ .map { case Row(label: Double, weight: Double, features: Vector) =>
+ Instance(label, weight, features)
+ }
+
+ if (familyObj == GeneralizedLinearRegression.Gaussian &&
+ linkObj == GeneralizedLinearRegression.Identity) {
+ // TODO: Make standardizeFeatures and standardizeLabel configurable.
+ val optimizer = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true)
+ val wlsModel = optimizer.fit(instances)
+ val model = copyValues(
+ new GeneralizedLinearRegressionModel(uid, wlsModel.coefficients, wlsModel.intercept)
+ .setParent(this))
+ return model
+ }
+
+ // Fit Generalized Linear Model by iteratively reweighted least squares (IRLS).
+ val initialModel = familyAndLink.initialize(instances, $(fitIntercept), $(regParam))
+ val optimizer = new IterativelyReweightedLeastSquares(initialModel, familyAndLink.reweightFunc,
+ $(fitIntercept), $(regParam), $(maxIter), $(tol))
+ val irlsModel = optimizer.fit(instances)
+
+ val model = copyValues(
+ new GeneralizedLinearRegressionModel(uid, irlsModel.coefficients, irlsModel.intercept)
+ .setParent(this))
+ model
+ }
+
+ @Since("2.0.0")
+ override def copy(extra: ParamMap): GeneralizedLinearRegression = defaultCopy(extra)
+}
+
+@Since("2.0.0")
+private[ml] object GeneralizedLinearRegression {
+
+ /** Set of family and link pairs that GeneralizedLinearRegression supports. */
+ lazy val supportedFamilyAndLinkParis = Set(
+ Gaussian -> Identity, Gaussian -> Log, Gaussian -> Inverse,
+ Binomial -> Logit, Binomial -> Probit, Binomial -> CLogLog,
+ Poisson -> Log, Poisson -> Identity, Poisson -> Sqrt,
+ Gamma -> Inverse, Gamma -> Identity, Gamma -> Log
+ )
+
+ /** Set of family names that GeneralizedLinearRegression supports. */
+ lazy val supportedFamilyNames = supportedFamilyAndLinkParis.map(_._1.name)
+
+ /** Set of link names that GeneralizedLinearRegression supports. */
+ lazy val supportedLinkNames = supportedFamilyAndLinkParis.map(_._2.name)
+
+ val epsilon: Double = 1E-16
+
+ /**
+ * Wrapper of family and link combination used in the model.
+ */
+ private[ml] class FamilyAndLink(val family: Family, var link: Link) extends Serializable {
--- End diff --
Should `link` be a val?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by AmplabJenkins <gi...@git.apache.org>.
Github user AmplabJenkins commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-181964083
Merged build finished. Test PASSed.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-183874403
**[Test build #51264 has finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/51264/consoleFull)** for PR 11136 at commit [`cc10147`](https://github.com/apache/spark/commit/cc101470bfd42b6fcabfe62ebf401ddf5d2bc1a7).
* This patch passes all tests.
* This patch merges cleanly.
* This patch adds no public classes.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by yanboliang <gi...@git.apache.org>.
Github user yanboliang commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r53913710
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,565 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionBase extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * Default is "gaussian".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "The name of family which is a description of the error distribution to be used in the " +
+ "model. Supported options: gaussian(default), binomial, poisson and gamma.",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilyNames.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of link function which provides the relationship
+ * between the linear predictor and the mean of the distribution function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "The name of link function " +
+ "which provides the relationship between the linear predictor and the mean of the " +
+ "distribution function. Supported options: identity, log, inverse, logit, probit, " +
+ "cloglog and sqrt.",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinkNames.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ import GeneralizedLinearRegression._
+ protected lazy val familyObj = Family.fromName($(family))
+ protected lazy val linkObj = if (isDefined(link)) {
+ Link.fromName($(link))
+ } else {
+ familyObj.defaultLink
+ }
+ protected lazy val familyAndLink = new FamilyAndLink(familyObj, linkObj)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ if ($(solver) == "irls") {
+ setDefault(maxIter -> 25)
+ }
+ if (isDefined(link)) {
+ require(GeneralizedLinearRegression.supportedFamilyAndLinkParis.contains(
+ familyObj -> linkObj), s"Generalized Linear Regression with ${$(family)} family " +
+ s"does not support ${$(link)} link function.")
+ }
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor (link function) and
+ * a description of the error distribution (family).
+ * It supports "gaussian", "binomial", "poisson" and "gamma" as family.
+ * Valid link functions for each family is listed below. The first link function of each family
+ * is the default one.
+ * - "gaussian" -> "identity", "log", "inverse"
+ * - "binomial" -> "logit", "probit", "cloglog"
+ * - "poisson" -> "log", "identity", "sqrt"
+ * - "gamma" -> "inverse", "identity", "log"
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionBase with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("glm"))
+
+ /**
+ * Sets the value of param [[family]].
+ * Default is "gaussian".
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+ setDefault(family -> "gaussian")
+
+ /**
+ * Sets the value of param [[link]].
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Sets if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+
+ /**
+ * Sets the maximum number of iterations.
+ * Default is 25 if the solver algorithm is "irls".
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+
+ /**
+ * Sets the convergence tolerance of iterations.
+ * Smaller value will lead to higher accuracy with the cost of more iterations.
+ * Default is 1E-6.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setTol(value: Double): this.type = set(tol, value)
+ setDefault(tol -> 1E-6)
--- End diff --
```tol``` has different semantics if we choose different solver(IRLS or LBFGS in the future). In our IRLS implementation, the convergence tolerance compared with [infinity norm] (https://github.com/apache/spark/blob/master/mllib/src/main/scala/org/apache/spark/ml/optim/IterativelyReweightedLeastSquares.scala#L92) of model vectors. If we support LBFGS, the convergence tolerance will compare with objective. R glm and glmnet have different convergence tolerance.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by yanboliang <gi...@git.apache.org>.
Github user yanboliang commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-188165176
@mengxr This PR is ready for another pass. Thanks!
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r53722399
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,547 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ if (isDefined(link)) {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
+ s"Generalized Linear Regression with ${$(family)} family does not support ${$(link)} " +
+ s"link function.")
+ }
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor and
+ * a description of the error distribution.
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("genLinReg"))
+
+ /**
+ * Set the name of family which is a description of the error distribution
--- End diff --
* `Set` -> `Sets`
* I don't think it is necessary to repeat the param doc. Maybe we can simply say `Sets the value of param [[family]].`
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by AmplabJenkins <gi...@git.apache.org>.
Github user AmplabJenkins commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-190538874
Test FAILed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/52216/
Test FAILed.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by yanboliang <gi...@git.apache.org>.
Github user yanboliang commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r54519070
--- Diff: mllib/src/test/scala/org/apache/spark/ml/regression/GeneralizedLinearRegressionSuite.scala ---
@@ -0,0 +1,499 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import scala.util.Random
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.ml.param.ParamsSuite
+import org.apache.spark.ml.util.MLTestingUtils
+import org.apache.spark.mllib.classification.LogisticRegressionSuite._
+import org.apache.spark.mllib.linalg.{BLAS, DenseVector, Vectors}
+import org.apache.spark.mllib.random._
+import org.apache.spark.mllib.regression.LabeledPoint
+import org.apache.spark.mllib.util.MLlibTestSparkContext
+import org.apache.spark.mllib.util.TestingUtils._
+import org.apache.spark.sql.{DataFrame, Row}
+
+class GeneralizedLinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
+
+ private val seed: Int = 42
+ @transient var datasetGaussianIdentity: DataFrame = _
+ @transient var datasetGaussianLog: DataFrame = _
+ @transient var datasetGaussianInverse: DataFrame = _
+ @transient var datasetBinomial: DataFrame = _
+ @transient var datasetPoissonLog: DataFrame = _
+ @transient var datasetPoissonIdentity: DataFrame = _
+ @transient var datasetPoissonSqrt: DataFrame = _
+ @transient var datasetGammaInverse: DataFrame = _
+ @transient var datasetGammaIdentity: DataFrame = _
+ @transient var datasetGammaLog: DataFrame = _
+
+ override def beforeAll(): Unit = {
+ super.beforeAll()
+
+ import GeneralizedLinearRegressionSuite._
+
+ datasetGaussianIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "identity"), 2))
+
+ datasetGaussianLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "log"), 2))
+
+ datasetGaussianInverse = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "inverse"), 2))
+
+ datasetBinomial = {
+ val nPoints = 10000
+ val coefficients = Array(-0.57997, 0.912083, -0.371077, -0.819866, 2.688191)
+ val xMean = Array(5.843, 3.057, 3.758, 1.199)
+ val xVariance = Array(0.6856, 0.1899, 3.116, 0.581)
+
+ val testData =
+ generateMultinomialLogisticInput(coefficients, xMean, xVariance, true, nPoints, seed)
+
+ sqlContext.createDataFrame(sc.parallelize(testData, 4))
+ }
+
+ datasetPoissonLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "log"), 2))
+
+ datasetPoissonIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "identity"), 2))
+
+ datasetPoissonSqrt = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "sqrt"), 2))
+
+ datasetGammaInverse = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "inverse"), 2))
+
+ datasetGammaIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "identity"), 2))
+
+ datasetGammaLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "log"), 2))
+ }
+
+ test("params") {
+ ParamsSuite.checkParams(new GeneralizedLinearRegression)
+ val model = new GeneralizedLinearRegressionModel("genLinReg", Vectors.dense(0.0), 0.0)
+ ParamsSuite.checkParams(model)
+ }
+
+ test("generalized linear regression: default params") {
+ val glr = new GeneralizedLinearRegression
+ assert(glr.getLabelCol === "label")
+ assert(glr.getFeaturesCol === "features")
+ assert(glr.getPredictionCol === "prediction")
+ assert(glr.getFitIntercept)
+ assert(glr.getTol === 1E-6)
+ assert(glr.getWeightCol === "")
+ assert(glr.getRegParam === 0.0)
+ assert(glr.getSolver == "irls")
+ val model = glr.setFamily("gaussian").setLink("identity")
+ .fit(datasetGaussianIdentity)
+
+ // copied model must have the same parent.
+ MLTestingUtils.checkCopy(model)
+ model.transform(datasetGaussianIdentity)
+ .select("label", "prediction")
+ .collect()
+
+ assert(model.getFeaturesCol === "features")
+ assert(model.getPredictionCol === "prediction")
+ assert(model.intercept !== 0.0)
+ assert(model.hasParent)
+ assert(model.getFamily === "gaussian")
+ assert(model.getLink === "identity")
+ }
+
+ test("generalized linear regression: gaussian family against glm") {
+ /*
+ R code:
+ f1 <- data$V1 ~ data$V2 + data$V3 - 1
+ f2 <- data$V1 ~ data$V2 + data$V3
+
+ data <- read.csv("path", header=FALSE)
+ for (formula in c(f1, f2)) {
+ model <- glm(formula, family="gaussian", data=data)
+ print(as.vector(coef(model)))
+ }
+
+ [1] 2.2960999 0.8087933
+ [1] 2.5002642 2.2000403 0.5999485
+
+ data <- read.csv("path", header=FALSE)
+ model1 <- glm(f1, family=gaussian(link=log), data=data, start=c(0,0))
+ model2 <- glm(f2, family=gaussian(link=log), data=data, start=c(0,0,0))
+ print(as.vector(coef(model1)))
+ print(as.vector(coef(model2)))
+
+ [1] 0.23069326 0.07993778
+ [1] 0.25001858 0.22002452 0.05998789
+
+ data <- read.csv("path", header=FALSE)
+ for (formula in c(f1, f2)) {
+ model <- glm(formula, family=gaussian(link=inverse), data=data)
+ print(as.vector(coef(model)))
+ }
+
+ [1] 2.3010179 0.8198976
+ [1] 2.4108902 2.2130248 0.6086152
+ */
+
+ val expected = Seq(
+ Vectors.dense(0.0, 2.2960999, 0.8087933),
+ Vectors.dense(2.5002642, 2.2000403, 0.5999485),
+ Vectors.dense(0.0, 0.23069326, 0.07993778),
+ Vectors.dense(0.25001858, 0.22002452, 0.05998789),
+ Vectors.dense(0.0, 2.3010179, 0.8198976),
+ Vectors.dense(2.4108902, 2.2130248, 0.6086152))
+
+ import GeneralizedLinearRegression._
+
+ var idx = 0
+ for ((link, dataset) <- Seq(("identity", datasetGaussianIdentity), ("log", datasetGaussianLog),
+ ("inverse", datasetGaussianInverse))) {
+ for (fitIntercept <- Seq(false, true)) {
+ val trainer = new GeneralizedLinearRegression().setFamily("gaussian").setLink(link)
+ .setFitIntercept(fitIntercept)
+ val model = trainer.fit(dataset)
+ val actual = Vectors.dense(model.intercept, model.coefficients(0), model.coefficients(1))
+ assert(actual ~== expected(idx) absTol 1e-4)
+
+ val familyLink = new FamilyAndLink(Gaussian, Link.fromName(link))
+ model.transform(dataset).select("features", "prediction").collect().foreach {
+ case Row(features: DenseVector, prediction1: Double) =>
+ val eta = BLAS.dot(features, model.coefficients) + model.intercept
+ val prediction2 = familyLink.fitted(eta)
+ assert(prediction1 ~== prediction2 relTol 1E-5)
--- End diff --
I agree. But there is a bug in the current ```TestingUtils``` which will block the error message always.
For example,
```Scala
assert(actual ~== expected(idx) absTol 1e-4, s"model mismatch: GLM gaussian family, $link link and fitIntercept = $fitIntercept.")
```
if ```actual``` not equal to ```expected(idx)``` under relative tolerance ```1e-4```, it will throw [exception](https://github.com/apache/spark/blob/master/mllib/src/test/scala/org/apache/spark/mllib/util/TestingUtils.scala#L133) first, so the error message specified by users will not be printed out.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r54059544
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,565 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionBase extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * Default is "gaussian".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "The name of family which is a description of the error distribution to be used in the " +
+ "model. Supported options: gaussian(default), binomial, poisson and gamma.",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilyNames.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of link function which provides the relationship
+ * between the linear predictor and the mean of the distribution function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "The name of link function " +
+ "which provides the relationship between the linear predictor and the mean of the " +
+ "distribution function. Supported options: identity, log, inverse, logit, probit, " +
+ "cloglog and sqrt.",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinkNames.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ import GeneralizedLinearRegression._
+ protected lazy val familyObj = Family.fromName($(family))
+ protected lazy val linkObj = if (isDefined(link)) {
+ Link.fromName($(link))
+ } else {
+ familyObj.defaultLink
+ }
+ protected lazy val familyAndLink = new FamilyAndLink(familyObj, linkObj)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ if ($(solver) == "irls") {
+ setDefault(maxIter -> 25)
+ }
+ if (isDefined(link)) {
+ require(GeneralizedLinearRegression.supportedFamilyAndLinkParis.contains(
+ familyObj -> linkObj), s"Generalized Linear Regression with ${$(family)} family " +
+ s"does not support ${$(link)} link function.")
+ }
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor (link function) and
+ * a description of the error distribution (family).
+ * It supports "gaussian", "binomial", "poisson" and "gamma" as family.
+ * Valid link functions for each family is listed below. The first link function of each family
+ * is the default one.
+ * - "gaussian" -> "identity", "log", "inverse"
+ * - "binomial" -> "logit", "probit", "cloglog"
+ * - "poisson" -> "log", "identity", "sqrt"
+ * - "gamma" -> "inverse", "identity", "log"
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionBase with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("glm"))
+
+ /**
+ * Sets the value of param [[family]].
+ * Default is "gaussian".
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+ setDefault(family -> "gaussian")
--- End diff --
`Gaussian.name`
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by AmplabJenkins <gi...@git.apache.org>.
Github user AmplabJenkins commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-181949574
Merged build finished. Test PASSed.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r54508383
--- Diff: mllib/src/test/scala/org/apache/spark/ml/regression/GeneralizedLinearRegressionSuite.scala ---
@@ -0,0 +1,499 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import scala.util.Random
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.ml.param.ParamsSuite
+import org.apache.spark.ml.util.MLTestingUtils
+import org.apache.spark.mllib.classification.LogisticRegressionSuite._
+import org.apache.spark.mllib.linalg.{BLAS, DenseVector, Vectors}
+import org.apache.spark.mllib.random._
+import org.apache.spark.mllib.regression.LabeledPoint
+import org.apache.spark.mllib.util.MLlibTestSparkContext
+import org.apache.spark.mllib.util.TestingUtils._
+import org.apache.spark.sql.{DataFrame, Row}
+
+class GeneralizedLinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
+
+ private val seed: Int = 42
+ @transient var datasetGaussianIdentity: DataFrame = _
+ @transient var datasetGaussianLog: DataFrame = _
+ @transient var datasetGaussianInverse: DataFrame = _
+ @transient var datasetBinomial: DataFrame = _
+ @transient var datasetPoissonLog: DataFrame = _
+ @transient var datasetPoissonIdentity: DataFrame = _
+ @transient var datasetPoissonSqrt: DataFrame = _
+ @transient var datasetGammaInverse: DataFrame = _
+ @transient var datasetGammaIdentity: DataFrame = _
+ @transient var datasetGammaLog: DataFrame = _
+
+ override def beforeAll(): Unit = {
+ super.beforeAll()
+
+ import GeneralizedLinearRegressionSuite._
+
+ datasetGaussianIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "identity"), 2))
+
+ datasetGaussianLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "log"), 2))
+
+ datasetGaussianInverse = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "inverse"), 2))
+
+ datasetBinomial = {
+ val nPoints = 10000
+ val coefficients = Array(-0.57997, 0.912083, -0.371077, -0.819866, 2.688191)
+ val xMean = Array(5.843, 3.057, 3.758, 1.199)
+ val xVariance = Array(0.6856, 0.1899, 3.116, 0.581)
+
+ val testData =
+ generateMultinomialLogisticInput(coefficients, xMean, xVariance, true, nPoints, seed)
+
+ sqlContext.createDataFrame(sc.parallelize(testData, 4))
+ }
+
+ datasetPoissonLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "log"), 2))
+
+ datasetPoissonIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "identity"), 2))
+
+ datasetPoissonSqrt = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "sqrt"), 2))
+
+ datasetGammaInverse = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "inverse"), 2))
+
+ datasetGammaIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "identity"), 2))
+
+ datasetGammaLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "log"), 2))
+ }
+
+ test("params") {
+ ParamsSuite.checkParams(new GeneralizedLinearRegression)
+ val model = new GeneralizedLinearRegressionModel("genLinReg", Vectors.dense(0.0), 0.0)
+ ParamsSuite.checkParams(model)
+ }
+
+ test("generalized linear regression: default params") {
+ val glr = new GeneralizedLinearRegression
+ assert(glr.getLabelCol === "label")
+ assert(glr.getFeaturesCol === "features")
+ assert(glr.getPredictionCol === "prediction")
+ assert(glr.getFitIntercept)
+ assert(glr.getTol === 1E-6)
+ assert(glr.getWeightCol === "")
+ assert(glr.getRegParam === 0.0)
+ assert(glr.getSolver == "irls")
+ val model = glr.setFamily("gaussian").setLink("identity")
+ .fit(datasetGaussianIdentity)
+
+ // copied model must have the same parent.
+ MLTestingUtils.checkCopy(model)
+ model.transform(datasetGaussianIdentity)
--- End diff --
What does this line test? The test name is `default params`.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by AmplabJenkins <gi...@git.apache.org>.
Github user AmplabJenkins commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-188138657
Merged build finished. Test PASSed.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by yanboliang <gi...@git.apache.org>.
Github user yanboliang commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-182905997
Jenkins, test this please.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r53722423
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,547 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ if (isDefined(link)) {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
+ s"Generalized Linear Regression with ${$(family)} family does not support ${$(link)} " +
+ s"link function.")
+ }
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor and
+ * a description of the error distribution.
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("genLinReg"))
+
+ /**
+ * Set the name of family which is a description of the error distribution
+ * to be used in the model.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
--- End diff --
Set the default to "gaussian"?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r53722623
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,547 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ if (isDefined(link)) {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
+ s"Generalized Linear Regression with ${$(family)} family does not support ${$(link)} " +
+ s"link function.")
+ }
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor and
+ * a description of the error distribution.
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("genLinReg"))
+
+ /**
+ * Set the name of family which is a description of the error distribution
+ * to be used in the model.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+
+ /**
+ * Set the name of the model link function.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Set if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+ setDefault(fitIntercept -> true)
+
+ /**
+ * Set the maximum number of iterations.
+ * Default is 100.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+ setDefault(maxIter -> 100)
+
+ /**
+ * Set the convergence tolerance of iterations.
+ * Smaller value will lead to higher accuracy with the cost of more iterations.
+ * Default is 1E-6.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setTol(value: Double): this.type = set(tol, value)
+ setDefault(tol -> 1E-6)
+
+ /**
+ * Set the regularization parameter.
+ * Default is 0.0.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setRegParam(value: Double): this.type = set(regParam, value)
+ setDefault(regParam -> 0.0)
+
+ /**
+ * Whether to over-/under-sample training instances according to the given weights in weightCol.
+ * If empty, all instances are treated equally (weight 1.0).
+ * Default is empty, so all instances have weight one.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setWeightCol(value: String): this.type = set(weightCol, value)
+ setDefault(weightCol -> "")
+
+ /**
+ * Set the solver algorithm used for optimization.
+ * Currently only support "irls" which is also the default solver.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setSolver(value: String): this.type = set(solver, value)
+ setDefault(solver -> "irls")
+
+ override protected def train(dataset: DataFrame): GeneralizedLinearRegressionModel = {
+ val familyLink = $(family) match {
+ case "gaussian" => if (isDefined(link)) Gaussian($(link)) else Gaussian("identity")
+ case "binomial" => if (isDefined(link)) Binomial($(link)) else Binomial("logit")
+ case "poisson" => if (isDefined(link)) Poisson($(link)) else Poisson("log")
+ case "gamma" => if (isDefined(link)) Gamma($(link)) else Gamma("inverse")
+ }
+
+ val numFeatures = dataset.select(col($(featuresCol))).limit(1).map {
+ case Row(features: Vector) => features.size
+ }.first()
+ if (numFeatures > 4096) {
+ val msg = "Currently, GeneralizedLinearRegression only supports number of features" +
+ s" <= 4096. Found $numFeatures in the input dataset."
+ throw new SparkException(msg)
+ }
+
+ val w = if ($(weightCol).isEmpty) lit(1.0) else col($(weightCol))
+ val instances: RDD[Instance] = dataset.select(
+ col($(labelCol)), w, col($(featuresCol))).map {
+ case Row(label: Double, weight: Double, features: Vector) =>
+ Instance(label, weight, features)
+ }
+
+ if ($(family) == "gaussian" && $(link) == "identity") {
+ val optimizer = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true)
+ val wlsModel = optimizer.fit(instances)
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ wlsModel.coefficients, wlsModel.intercept).setParent(this))
--- End diff --
better alignment, e.g.:
~~~scala
val model = copyValues(
new GeneralizedLinearRegressionModel(uid, wlsModel.coefficients, wlsModel.intercept)
.setParent(this))
~~~
The problem is `setParent` is in the same line as partial arguments from the constructors.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r53722703
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,547 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ if (isDefined(link)) {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
+ s"Generalized Linear Regression with ${$(family)} family does not support ${$(link)} " +
+ s"link function.")
+ }
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor and
+ * a description of the error distribution.
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("genLinReg"))
+
+ /**
+ * Set the name of family which is a description of the error distribution
+ * to be used in the model.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+
+ /**
+ * Set the name of the model link function.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Set if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+ setDefault(fitIntercept -> true)
+
+ /**
+ * Set the maximum number of iterations.
+ * Default is 100.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+ setDefault(maxIter -> 100)
+
+ /**
+ * Set the convergence tolerance of iterations.
+ * Smaller value will lead to higher accuracy with the cost of more iterations.
+ * Default is 1E-6.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setTol(value: Double): this.type = set(tol, value)
+ setDefault(tol -> 1E-6)
+
+ /**
+ * Set the regularization parameter.
+ * Default is 0.0.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setRegParam(value: Double): this.type = set(regParam, value)
+ setDefault(regParam -> 0.0)
+
+ /**
+ * Whether to over-/under-sample training instances according to the given weights in weightCol.
+ * If empty, all instances are treated equally (weight 1.0).
+ * Default is empty, so all instances have weight one.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setWeightCol(value: String): this.type = set(weightCol, value)
+ setDefault(weightCol -> "")
+
+ /**
+ * Set the solver algorithm used for optimization.
+ * Currently only support "irls" which is also the default solver.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setSolver(value: String): this.type = set(solver, value)
+ setDefault(solver -> "irls")
+
+ override protected def train(dataset: DataFrame): GeneralizedLinearRegressionModel = {
+ val familyLink = $(family) match {
+ case "gaussian" => if (isDefined(link)) Gaussian($(link)) else Gaussian("identity")
+ case "binomial" => if (isDefined(link)) Binomial($(link)) else Binomial("logit")
+ case "poisson" => if (isDefined(link)) Poisson($(link)) else Poisson("log")
+ case "gamma" => if (isDefined(link)) Gamma($(link)) else Gamma("inverse")
+ }
+
+ val numFeatures = dataset.select(col($(featuresCol))).limit(1).map {
+ case Row(features: Vector) => features.size
+ }.first()
+ if (numFeatures > 4096) {
+ val msg = "Currently, GeneralizedLinearRegression only supports number of features" +
+ s" <= 4096. Found $numFeatures in the input dataset."
+ throw new SparkException(msg)
+ }
+
+ val w = if ($(weightCol).isEmpty) lit(1.0) else col($(weightCol))
+ val instances: RDD[Instance] = dataset.select(
+ col($(labelCol)), w, col($(featuresCol))).map {
+ case Row(label: Double, weight: Double, features: Vector) =>
+ Instance(label, weight, features)
+ }
+
+ if ($(family) == "gaussian" && $(link) == "identity") {
+ val optimizer = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true)
+ val wlsModel = optimizer.fit(instances)
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ wlsModel.coefficients, wlsModel.intercept).setParent(this))
+ return model
+ }
+
+ val newInstances = instances.map { instance =>
+ val mu = familyLink.initialize(instance.label, instance.weight)
+ val eta = familyLink.predict(mu)
+ Instance(eta, instance.weight, instance.features)
+ }
+
+ val initialModel = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true).fit(newInstances)
+
+ val reweightFunc: (Instance, WeightedLeastSquaresModel) => (Double, Double) = {
+ (instance: Instance, model: WeightedLeastSquaresModel) => {
+ val eta = model.predict(instance.features)
+ val mu = familyLink.fitted(eta)
+ val z = familyLink.adjusted(instance.label, mu, eta)
+ val w = familyLink.weights(mu) * instance.weight
+ (z, w)
+ }
+ }
+
+ val optimizer = new IterativelyReweightedLeastSquares(initialModel, reweightFunc,
+ $(fitIntercept), $(regParam), $(maxIter), $(tol))
+
+ val irlsModel = optimizer.fit(instances)
+
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ irlsModel.coefficients, irlsModel.intercept).setParent(this))
+ model
+ }
+
+ @Since("2.0.0")
+ override def copy(extra: ParamMap): GeneralizedLinearRegression = defaultCopy(extra)
+}
+
+@Since("2.0.0")
+object GeneralizedLinearRegression {
+
+ /** Set of families that GeneralizedLinearRegression supports */
+ private[ml] val supportedFamilies = Set("gaussian", "binomial", "poisson", "gamma")
+
+ /** Set of links that GeneralizedLinearRegression supports */
+ private[ml] val supportedLinks = Set("identity", "log", "inverse", "logit", "probit",
+ "cloglog", "sqrt")
+
+ /** Set of family and link pairs that GeneralizedLinearRegression supports */
+ private[ml] val supportedFamilyLinkPairs = Set(
+ "gaussian" -> "identity", "gaussian" -> "log", "gaussian" -> "inverse",
+ "binomial" -> "logit", "binomial" -> "probit", "binomial" -> "cloglog",
+ "poisson" -> "log", "poisson" -> "identity", "poisson" -> "sqrt",
+ "gamma" -> "inverse", "gamma" -> "identity", "gamma" -> "log"
+ )
+}
+
+/**
+ * :: Experimental ::
+ * Model produced by [[GeneralizedLinearRegression]].
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegressionModel private[ml] (
+ @Since("2.0.0") override val uid: String,
+ @Since("2.0.0") val coefficients: Vector,
+ @Since("2.0.0") val intercept: Double)
+ extends RegressionModel[Vector, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams {
+
+ private lazy val familyLink = $(family) match {
+ case "gaussian" => if (isDefined(link)) Gaussian($(link)) else Gaussian("identity")
+ case "binomial" => if (isDefined(link)) Binomial($(link)) else Binomial("logit")
+ case "poisson" => if (isDefined(link)) Poisson($(link)) else Poisson("log")
+ case "gamma" => if (isDefined(link)) Gamma($(link)) else Gamma("inverse")
+ }
+
+ override protected def predict(features: Vector): Double = {
+ val eta = BLAS.dot(features, coefficients) + intercept
+ familyLink.fitted(eta)
+ }
+
+ @Since("2.0.0")
+ override def copy(extra: ParamMap): GeneralizedLinearRegressionModel = {
+ copyValues(new GeneralizedLinearRegressionModel(uid, coefficients, intercept), extra)
+ .setParent(parent)
+ }
+}
+
+/**
+ * A description of the error distribution and link function to be used in the model.
+ * @param link a link function instance
+ */
+private[ml] abstract class Family(val link: Link) extends Serializable {
+
+ /** Initialize the starting value for mu. */
+ def initialize(y: Double, weight: Double): Double
+
+ /** The variance of the endogenous variable's mean, given the value mu. */
+ def variance(mu: Double): Double
+
+ /** Weights for IRLS steps. */
+ def weights(mu: Double): Double = {
+ val x = clean(mu)
+ 1.0 / (math.pow(this.link.deriv(x), 2.0) * this.variance(x))
+ }
+
+ /** The adjusted response variable. */
+ def adjusted(y: Double, mu: Double, eta: Double): Double = {
+ val x = clean(mu)
+ eta + (y - x) * link.deriv(x)
+ }
+
+ /** Linear predictor based on given mu. */
+ def predict(mu: Double): Double = this.link.link(clean(mu))
+
+ /** Fitted value based on linear predictor eta. */
+ def fitted(eta: Double): Double = clean(this.link.unlink(eta))
+
+ /** Trim the fitted value so that it will be in valid range. */
+ def clean(mu: Double): Double = mu
+
+ val epsilon: Double = 1E-16
+}
+
+/**
+ * Gaussian exponential family distribution.
+ * The default link for the Gaussian family is the identity link.
+ * @param link a link function instance
+ */
+private[ml] class Gaussian(link: Link = Identity) extends Family(link) {
+
+ override def initialize(y: Double, weight: Double): Double = {
+ if (link == Log) {
+ require(y > 0.0, "The response variable of Gaussian family with Log link " +
+ s"should be positive, but got $y")
+ }
+ y
+ }
+
+ def variance(mu: Double): Double = 1.0
+
+ override def clean(mu: Double): Double = {
+ if (mu.isNegInfinity) {
+ Double.MinValue
+ } else if (mu.isPosInfinity) {
+ Double.MaxValue
+ } else {
+ mu
+ }
+ }
+}
+
+private[ml] object Gaussian {
+
+ def apply(link: String): Gaussian = {
+ link match {
+ case "identity" => new Gaussian(Identity)
+ case "log" => new Gaussian(Log)
+ case "inverse" => new Gaussian(Inverse)
+ }
+ }
+}
+
+/**
+ * Binomial exponential family distribution.
+ * The default link for the Binomial family is the logit link.
+ * @param link a link function instance
+ */
+private[ml] class Binomial(link: Link = Logit) extends Family(link) {
+
+ override def initialize(y: Double, weight: Double): Double = {
+ val mu = (weight * y + 0.5) / (weight + 1.0)
+ require(mu > 0.0 && mu < 1.0, "The response variable of Binomial family" +
+ s"should be in range (0, 1), but got $mu")
+ mu
+ }
+
+ override def variance(mu: Double): Double = mu * (1.0 - mu)
+
+ override def clean(mu: Double): Double = {
+ if (mu < epsilon) {
+ epsilon
+ } else if (mu > 1.0 - epsilon) {
+ 1.0 - epsilon
+ } else {
+ mu
+ }
+ }
+}
+
+private[ml] object Binomial {
+
+ def apply(link: String): Binomial = {
+ link match {
+ case "logit" => new Binomial(Logit)
+ case "probit" => new Binomial(Probit)
+ case "cloglog" => new Binomial(CLogLog)
+ }
+ }
+}
+
+/**
+ * Poisson exponential family distribution.
+ * The default link for the Poisson family is the log link.
+ * @param link a link function instance
+ */
+private[ml] class Poisson(link: Link = Log) extends Family(link) {
+
+ override def initialize(y: Double, weight: Double): Double = {
+ require(y > 0.0, "The response variable of Poisson family " +
+ s"should be positive, but got $y")
+ y
+ }
+
+ override def variance(mu: Double): Double = mu
+
+ override def clean(mu: Double): Double = {
+ if (mu < epsilon) {
+ epsilon
+ } else if (mu.isInfinity) {
+ Double.MaxValue
+ } else {
+ mu
+ }
+ }
+}
+
+private[ml] object Poisson {
+
+ def apply(link: String): Poisson = {
+ link match {
+ case "log" => new Poisson(Log)
+ case "sqrt" => new Poisson(Sqrt)
+ case "identity" => new Poisson(Identity)
+ }
+ }
+}
+
+/**
+ * Gamma exponential family distribution.
+ * The default link for the Gamma family is the inverse link.
+ * @param link a link function instance
+ */
+private[ml] class Gamma(link: Link = Inverse) extends Family(link) {
--- End diff --
Looking through the family implementations, I think we could decouple `Family` from `Link` and create a new class called `FamilyAndLink`. `Family` handles init, variance, and clean, while `FamilyAndLink` handles reweight and adjust. This could simplify the code. We can have `Family.fromName` and `Link.fromName` to handle raw strings and use objects in our implementation.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-183867020
**[Test build #51264 has started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/51264/consoleFull)** for PR 11136 at commit [`cc10147`](https://github.com/apache/spark/commit/cc101470bfd42b6fcabfe62ebf401ddf5d2bc1a7).
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by yanboliang <gi...@git.apache.org>.
Github user yanboliang commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r52611868
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,472 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.Logging
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
+ s"Generalized Linear Regression with ${$(family)} family does not support ${$(link)} " +
+ s"link function.")
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor and
+ * a description of the error distribution.
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("genLinReg"))
+
+ /**
+ * Set the name of family which is a description of the error distribution
+ * to be used in the model.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+
+ /**
+ * Set the name of the model link function.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Set if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+ setDefault(fitIntercept -> true)
+
+ /**
+ * Set the maximum number of iterations.
+ * Default is 100.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+ setDefault(maxIter -> 100)
+
+ /**
+ * Set the convergence tolerance of iterations.
+ * Smaller value will lead to higher accuracy with the cost of more iterations.
+ * Default is 1E-6.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setTol(value: Double): this.type = set(tol, value)
+ setDefault(tol -> 1E-6)
+
+ /**
+ * Set the regularization parameter.
+ * Default is 0.0.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setRegParam(value: Double): this.type = set(regParam, value)
+ setDefault(regParam -> 0.0)
+
+ /**
+ * Whether to over-/under-sample training instances according to the given weights in weightCol.
+ * If empty, all instances are treated equally (weight 1.0).
+ * Default is empty, so all instances have weight one.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setWeightCol(value: String): this.type = set(weightCol, value)
+ setDefault(weightCol -> "")
+
+ /**
+ * Set the solver algorithm used for optimization.
+ * Currently only support "irls" which is also the default solver.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setSolver(value: String): this.type = set(solver, value)
+ setDefault(solver -> "irls")
+
+ override protected def train(dataset: DataFrame): GeneralizedLinearRegressionModel = {
--- End diff --
Yep, I have add check for the number of features at the start of training and throw exceptions if it great than 4096.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by sethah <gi...@git.apache.org>.
Github user sethah commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r52400669
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,472 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.Logging
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
+ s"Generalized Linear Regression with ${$(family)} family does not support ${$(link)} " +
+ s"link function.")
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor and
+ * a description of the error distribution.
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("genLinReg"))
+
+ /**
+ * Set the name of family which is a description of the error distribution
+ * to be used in the model.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+
+ /**
+ * Set the name of the model link function.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Set if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+ setDefault(fitIntercept -> true)
+
+ /**
+ * Set the maximum number of iterations.
+ * Default is 100.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+ setDefault(maxIter -> 100)
+
+ /**
+ * Set the convergence tolerance of iterations.
+ * Smaller value will lead to higher accuracy with the cost of more iterations.
+ * Default is 1E-6.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setTol(value: Double): this.type = set(tol, value)
+ setDefault(tol -> 1E-6)
+
+ /**
+ * Set the regularization parameter.
+ * Default is 0.0.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setRegParam(value: Double): this.type = set(regParam, value)
+ setDefault(regParam -> 0.0)
+
+ /**
+ * Whether to over-/under-sample training instances according to the given weights in weightCol.
+ * If empty, all instances are treated equally (weight 1.0).
+ * Default is empty, so all instances have weight one.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setWeightCol(value: String): this.type = set(weightCol, value)
+ setDefault(weightCol -> "")
+
+ /**
+ * Set the solver algorithm used for optimization.
+ * Currently only support "irls" which is also the default solver.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setSolver(value: String): this.type = set(solver, value)
+ setDefault(solver -> "irls")
+
+ override protected def train(dataset: DataFrame): GeneralizedLinearRegressionModel = {
+ val familyLink = $(family) match {
+ case "gaussian" => Gaussian($(link))
+ case "binomial" => Binomial($(link))
+ case "poisson" => Poisson($(link))
+ case "gamma" => Gamma($(link))
+ }
+
+ val w = if ($(weightCol).isEmpty) lit(1.0) else col($(weightCol))
+ val instances: RDD[Instance] = dataset.select(
+ col($(labelCol)), w, col($(featuresCol))).map {
+ case Row(label: Double, weight: Double, features: Vector) =>
+ Instance(label, weight, features)
+ }
+
+ if ($(family) == "gaussian" && $(link) == "identity") {
+ val optimizer = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true)
+ val wlsModel = optimizer.fit(instances)
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ wlsModel.coefficients, wlsModel.intercept).setParent(this))
+ return model
+ }
+
+ val newInstances = instances.map { instance =>
+ val mu = familyLink.initialize(instance.label, instance.weight)
+ val eta = familyLink.predict(mu)
+ Instance(eta, instance.weight, instance.features)
+ }
+
+ val initialModel = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true).fit(newInstances)
+
+ val reweightFunc: (Instance, WeightedLeastSquaresModel) => (Double, Double) = {
+ (instance: Instance, model: WeightedLeastSquaresModel) => {
+ val eta = model.predict(instance.features)
+ val mu = familyLink.fitted(eta)
+ val z = familyLink.adjusted(instance.label, mu, eta)
+ val w = familyLink.weights(mu) * instance.weight
+ (z, w)
+ }
+ }
+
+ val optimizer = new IterativelyReweightedLeastSquares(initialModel, reweightFunc,
+ $(fitIntercept), $(regParam), $(maxIter), $(tol))
+
+ val irlsModel = optimizer.fit(instances)
+
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ irlsModel.coefficients, irlsModel.intercept).setParent(this))
+ model
+ }
+
+ @Since("2.0.0")
+ override def copy(extra: ParamMap): GeneralizedLinearRegression = defaultCopy(extra)
+}
+
+@Since("2.0.0")
+object GeneralizedLinearRegression {
+
+ /** Set of families that GeneralizedLinearRegression supports */
+ private[ml] val supportedFamilies = Set("gaussian", "binomial", "poisson", "gamma")
+
+ /** Set of links that GeneralizedLinearRegression supports */
+ private[ml] val supportedLinks = Set("identity", "log", "inverse", "logit", "probit",
+ "cloglog", "sqrt")
+
+ /** Set of family and link pairs that GeneralizedLinearRegression supports */
+ private[ml] val supportedFamilyLinkPairs = Set(
+ "gaussian" -> "identity", "gaussian" -> "log", "gaussian" -> "inverse",
+ "binomial" -> "logit", "binomial" -> "probit", "binomial" -> "cloglog",
+ "poisson" -> "log", "poisson" -> "identity", "poisson" -> "sqrt",
+ "gamma" -> "inverse", "gamma" -> "identity", "gamma" -> "log"
+ )
+}
+
+/**
+ * :: Experimental ::
+ * Model produced by [[GeneralizedLinearRegression]].
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegressionModel private[ml] (
+ @Since("2.0.0") override val uid: String,
+ @Since("2.0.0") val coefficients: Vector,
+ @Since("2.0.0") val intercept: Double)
+ extends RegressionModel[Vector, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams {
+
+ private lazy val familyLink = $(family) match {
+ case "gaussian" => Gaussian($(link))
+ case "binomial" => Binomial($(link))
+ case "poisson" => Poisson($(link))
+ case "gamma" => Gamma($(link))
+ }
+
+ override protected def predict(features: Vector): Double = {
+ val eta = BLAS.dot(features, coefficients) + intercept
+ familyLink.fitted(eta)
+ }
+
+ @Since("2.0.0")
+ override def copy(extra: ParamMap): GeneralizedLinearRegressionModel = {
+ copyValues(new GeneralizedLinearRegressionModel(uid, coefficients, intercept), extra)
+ .setParent(parent)
+ }
+}
+
+/**
+ * A description of the error distribution and link function to be used in the model.
+ * @param link a link function instance
+ */
+private[ml] abstract class Family(val link: Link) extends Serializable {
+
+ /** Initialize the starting value for mu. */
+ def initialize(y: Double, weight: Double): Double
+
+ /** The variance of mu to its mean. */
+ def variance(mu: Double): Double
+
+ /** Weights for IRLS steps. */
+ def weights(mu: Double): Double = {
+ 1.0 / (math.pow(this.link.deriv(mu), 2.0) * this.variance(mu))
+ }
+
+ /** The adjusted response variable. */
+ def adjusted(y: Double, mu: Double, eta: Double): Double = {
+ eta + (y - mu) * link.deriv(mu)
+ }
+
+ /** Linear predictors based on given mu. */
+ def predict(mu: Double): Double = this.link.link(mu)
+
+ /** Fitted values based on linear predictors eta. */
+ def fitted(eta: Double): Double = this.link.unlink(eta)
+}
+
+/**
+ * Gaussian exponential family distribution.
+ * The default link for the Gaussian family is the identity link.
+ * @param link a link function instance
+ */
+private[ml] class Gaussian(link: Link = Identity) extends Family(link) {
+
+ override def initialize(y: Double, weight: Double): Double = y
+
+ def variance(mu: Double): Double = 1.0
+}
+
+private[ml] object Gaussian {
+
+ def apply(link: String): Gaussian = {
+ link match {
+ case "identity" => new Gaussian(Identity)
+ case "log" => new Gaussian(Log)
+ case "inverse" => new Gaussian(Inverse)
+ }
+ }
+}
+
+/**
+ * Binomial exponential family distribution.
+ * The default link for the Binomial family is the logit link.
+ * @param link a link function instance
+ */
+private[ml] class Binomial(link: Link = Logit) extends Family(link) {
+
+ override def initialize(y: Double, weight: Double): Double = {
+ (weight * y + 0.5) / (weight + 1.0)
+ }
+
+ override def variance(mu: Double): Double = mu * (1 - mu)
+}
+
+private[ml] object Binomial {
+
+ def apply(link: String): Binomial = {
+ link match {
+ case "logit" => new Binomial(Logit)
+ case "probit" => new Binomial(Probit)
+ case "cloglog" => new Binomial(CLogLog)
+ }
+ }
+}
+
+/**
+ * Poisson exponential family distribution.
+ * The default link for the Poisson family is the log link.
+ * @param link a link function instance
+ */
+private[ml] class Poisson(link: Link = Log) extends Family(link) {
+
+ override def initialize(y: Double, weight: Double): Double = y + 0.1
+
+ override def variance(mu: Double): Double = mu
+}
+
+private[ml] object Poisson {
+
+ def apply(link: String): Poisson = {
+ link match {
+ case "log" => new Poisson(Log)
+ case "sqrt" => new Poisson(Sqrt)
+ case "identity" => new Poisson(Identity)
+ }
+ }
+}
+
+/**
+ * Gamma exponential family distribution.
+ * The default link for the Gamma family is the inverse link.
+ * @param link a link function instance
+ */
+private[ml] class Gamma(link: Link = Log) extends Family(link) {
+
+ override def initialize(y: Double, weight: Double): Double = y
+
+ override def variance(mu: Double): Double = math.pow(mu, 2.0)
+}
+
+private[ml] object Gamma {
+
+ def apply(link: String): Gamma = {
+ link match {
+ case "inverse" => new Gamma(Inverse)
+ case "identity" => new Gamma(Identity)
+ case "log" => new Gamma(Log)
+ }
+ }
+}
+
+/**
+ * A description of the link function to be used in the model.
+ */
+private[ml] trait Link extends Serializable {
+
+ /** The link function. */
+ def link(mu: Double): Double
+
+ /** The derivative function. */
--- End diff --
nit: "Derivative of the link function".
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-188662047
Only some minor comments on the implementation. I will make a pass on the tests tomorrow. @dbtsai It would be great if you can make a pass too.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-188745464
**[Test build #51963 has finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/51963/consoleFull)** for PR 11136 at commit [`2ebcef7`](https://github.com/apache/spark/commit/2ebcef728315ad6b03d48c7f7e7f504e5e193748).
* This patch passes all tests.
* This patch merges cleanly.
* This patch adds no public classes.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-187443051
I'm making a pass.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r54508437
--- Diff: mllib/src/test/scala/org/apache/spark/ml/regression/GeneralizedLinearRegressionSuite.scala ---
@@ -0,0 +1,499 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import scala.util.Random
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.ml.param.ParamsSuite
+import org.apache.spark.ml.util.MLTestingUtils
+import org.apache.spark.mllib.classification.LogisticRegressionSuite._
+import org.apache.spark.mllib.linalg.{BLAS, DenseVector, Vectors}
+import org.apache.spark.mllib.random._
+import org.apache.spark.mllib.regression.LabeledPoint
+import org.apache.spark.mllib.util.MLlibTestSparkContext
+import org.apache.spark.mllib.util.TestingUtils._
+import org.apache.spark.sql.{DataFrame, Row}
+
+class GeneralizedLinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
+
+ private val seed: Int = 42
+ @transient var datasetGaussianIdentity: DataFrame = _
+ @transient var datasetGaussianLog: DataFrame = _
+ @transient var datasetGaussianInverse: DataFrame = _
+ @transient var datasetBinomial: DataFrame = _
+ @transient var datasetPoissonLog: DataFrame = _
+ @transient var datasetPoissonIdentity: DataFrame = _
+ @transient var datasetPoissonSqrt: DataFrame = _
+ @transient var datasetGammaInverse: DataFrame = _
+ @transient var datasetGammaIdentity: DataFrame = _
+ @transient var datasetGammaLog: DataFrame = _
+
+ override def beforeAll(): Unit = {
+ super.beforeAll()
+
+ import GeneralizedLinearRegressionSuite._
+
+ datasetGaussianIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "identity"), 2))
+
+ datasetGaussianLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "log"), 2))
+
+ datasetGaussianInverse = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "inverse"), 2))
+
+ datasetBinomial = {
+ val nPoints = 10000
+ val coefficients = Array(-0.57997, 0.912083, -0.371077, -0.819866, 2.688191)
+ val xMean = Array(5.843, 3.057, 3.758, 1.199)
+ val xVariance = Array(0.6856, 0.1899, 3.116, 0.581)
+
+ val testData =
+ generateMultinomialLogisticInput(coefficients, xMean, xVariance, true, nPoints, seed)
+
+ sqlContext.createDataFrame(sc.parallelize(testData, 4))
+ }
+
+ datasetPoissonLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "log"), 2))
+
+ datasetPoissonIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "identity"), 2))
+
+ datasetPoissonSqrt = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "sqrt"), 2))
+
+ datasetGammaInverse = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "inverse"), 2))
+
+ datasetGammaIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "identity"), 2))
+
+ datasetGammaLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "log"), 2))
+ }
+
+ test("params") {
+ ParamsSuite.checkParams(new GeneralizedLinearRegression)
+ val model = new GeneralizedLinearRegressionModel("genLinReg", Vectors.dense(0.0), 0.0)
+ ParamsSuite.checkParams(model)
+ }
+
+ test("generalized linear regression: default params") {
+ val glr = new GeneralizedLinearRegression
+ assert(glr.getLabelCol === "label")
+ assert(glr.getFeaturesCol === "features")
+ assert(glr.getPredictionCol === "prediction")
+ assert(glr.getFitIntercept)
+ assert(glr.getTol === 1E-6)
+ assert(glr.getWeightCol === "")
+ assert(glr.getRegParam === 0.0)
+ assert(glr.getSolver == "irls")
+ val model = glr.setFamily("gaussian").setLink("identity")
+ .fit(datasetGaussianIdentity)
+
+ // copied model must have the same parent.
+ MLTestingUtils.checkCopy(model)
+ model.transform(datasetGaussianIdentity)
+ .select("label", "prediction")
+ .collect()
+
+ assert(model.getFeaturesCol === "features")
+ assert(model.getPredictionCol === "prediction")
+ assert(model.intercept !== 0.0)
+ assert(model.hasParent)
+ assert(model.getFamily === "gaussian")
+ assert(model.getLink === "identity")
+ }
+
+ test("generalized linear regression: gaussian family against glm") {
+ /*
+ R code:
+ f1 <- data$V1 ~ data$V2 + data$V3 - 1
+ f2 <- data$V1 ~ data$V2 + data$V3
+
+ data <- read.csv("path", header=FALSE)
+ for (formula in c(f1, f2)) {
+ model <- glm(formula, family="gaussian", data=data)
+ print(as.vector(coef(model)))
+ }
+
+ [1] 2.2960999 0.8087933
+ [1] 2.5002642 2.2000403 0.5999485
+
+ data <- read.csv("path", header=FALSE)
+ model1 <- glm(f1, family=gaussian(link=log), data=data, start=c(0,0))
+ model2 <- glm(f2, family=gaussian(link=log), data=data, start=c(0,0,0))
+ print(as.vector(coef(model1)))
+ print(as.vector(coef(model2)))
+
+ [1] 0.23069326 0.07993778
+ [1] 0.25001858 0.22002452 0.05998789
+
+ data <- read.csv("path", header=FALSE)
+ for (formula in c(f1, f2)) {
+ model <- glm(formula, family=gaussian(link=inverse), data=data)
+ print(as.vector(coef(model)))
+ }
+
+ [1] 2.3010179 0.8198976
+ [1] 2.4108902 2.2130248 0.6086152
+ */
+
+ val expected = Seq(
+ Vectors.dense(0.0, 2.2960999, 0.8087933),
+ Vectors.dense(2.5002642, 2.2000403, 0.5999485),
+ Vectors.dense(0.0, 0.23069326, 0.07993778),
+ Vectors.dense(0.25001858, 0.22002452, 0.05998789),
+ Vectors.dense(0.0, 2.3010179, 0.8198976),
+ Vectors.dense(2.4108902, 2.2130248, 0.6086152))
+
+ import GeneralizedLinearRegression._
+
+ var idx = 0
+ for ((link, dataset) <- Seq(("identity", datasetGaussianIdentity), ("log", datasetGaussianLog),
+ ("inverse", datasetGaussianInverse))) {
+ for (fitIntercept <- Seq(false, true)) {
+ val trainer = new GeneralizedLinearRegression().setFamily("gaussian").setLink(link)
+ .setFitIntercept(fitIntercept)
+ val model = trainer.fit(dataset)
+ val actual = Vectors.dense(model.intercept, model.coefficients(0), model.coefficients(1))
+ assert(actual ~== expected(idx) absTol 1e-4)
+
+ val familyLink = new FamilyAndLink(Gaussian, Link.fromName(link))
+ model.transform(dataset).select("features", "prediction").collect().foreach {
+ case Row(features: DenseVector, prediction1: Double) =>
+ val eta = BLAS.dot(features, model.coefficients) + model.intercept
+ val prediction2 = familyLink.fitted(eta)
+ assert(prediction1 ~== prediction2 relTol 1E-5)
+ }
+
+ idx += 1
+ }
+ }
+ }
+
+ test("generalized linear regression: gaussian family against glmnet") {
+ /*
+ R code:
+ library(glmnet)
+ data <- read.csv("path", header=FALSE)
+ label = data$V1
+ features = as.matrix(data.frame(data$V2, data$V3))
+ for (intercept in c(FALSE, TRUE)) {
+ for (lambda in c(0.0, 0.1, 1.0)) {
+ model <- glmnet(features, label, family="gaussian", intercept=intercept,
+ lambda=lambda, alpha=0, thresh=1E-14)
+ print(as.vector(coef(model)))
+ }
+ }
+
+ [1] 0.0000000 2.2961005 0.8087932
+ [1] 0.0000000 2.2130368 0.8309556
+ [1] 0.0000000 1.7176137 0.9610657
+ [1] 2.5002642 2.2000403 0.5999485
+ [1] 3.1106389 2.0935142 0.5712711
+ [1] 6.7597127 1.4581054 0.3994266
+ */
+
+ val expected = Seq(
+ Vectors.dense(0.0, 2.2961005, 0.8087932),
+ Vectors.dense(0.0, 2.2130368, 0.8309556),
+ Vectors.dense(0.0, 1.7176137, 0.9610657),
+ Vectors.dense(2.5002642, 2.2000403, 0.5999485),
+ Vectors.dense(3.1106389, 2.0935142, 0.5712711),
+ Vectors.dense(6.7597127, 1.4581054, 0.3994266))
+
+ var idx = 0
+ for (fitIntercept <- Seq(false, true);
+ regParam <- Seq(0.0, 0.1, 1.0)) {
+ val trainer = new GeneralizedLinearRegression().setFamily("gaussian")
+ .setFitIntercept(fitIntercept).setRegParam(regParam)
+ val model = trainer.fit(datasetGaussianIdentity)
+ val actual = Vectors.dense(model.intercept, model.coefficients(0), model.coefficients(1))
+ assert(actual ~== expected(idx) absTol 1e-4)
--- End diff --
ditto
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r53722512
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,547 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ if (isDefined(link)) {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
+ s"Generalized Linear Regression with ${$(family)} family does not support ${$(link)} " +
+ s"link function.")
+ }
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor and
+ * a description of the error distribution.
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("genLinReg"))
+
+ /**
+ * Set the name of family which is a description of the error distribution
+ * to be used in the model.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+
+ /**
+ * Set the name of the model link function.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Set if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+ setDefault(fitIntercept -> true)
+
+ /**
+ * Set the maximum number of iterations.
+ * Default is 100.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+ setDefault(maxIter -> 100)
+
+ /**
+ * Set the convergence tolerance of iterations.
+ * Smaller value will lead to higher accuracy with the cost of more iterations.
+ * Default is 1E-6.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setTol(value: Double): this.type = set(tol, value)
+ setDefault(tol -> 1E-6)
+
+ /**
+ * Set the regularization parameter.
+ * Default is 0.0.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setRegParam(value: Double): this.type = set(regParam, value)
+ setDefault(regParam -> 0.0)
+
+ /**
+ * Whether to over-/under-sample training instances according to the given weights in weightCol.
+ * If empty, all instances are treated equally (weight 1.0).
+ * Default is empty, so all instances have weight one.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setWeightCol(value: String): this.type = set(weightCol, value)
+ setDefault(weightCol -> "")
+
+ /**
+ * Set the solver algorithm used for optimization.
+ * Currently only support "irls" which is also the default solver.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setSolver(value: String): this.type = set(solver, value)
+ setDefault(solver -> "irls")
+
+ override protected def train(dataset: DataFrame): GeneralizedLinearRegressionModel = {
+ val familyLink = $(family) match {
+ case "gaussian" => if (isDefined(link)) Gaussian($(link)) else Gaussian("identity")
+ case "binomial" => if (isDefined(link)) Binomial($(link)) else Binomial("logit")
+ case "poisson" => if (isDefined(link)) Poisson($(link)) else Poisson("log")
+ case "gamma" => if (isDefined(link)) Gamma($(link)) else Gamma("inverse")
+ }
+
+ val numFeatures = dataset.select(col($(featuresCol))).limit(1).map {
+ case Row(features: Vector) => features.size
+ }.first()
+ if (numFeatures > 4096) {
+ val msg = "Currently, GeneralizedLinearRegression only supports number of features" +
+ s" <= 4096. Found $numFeatures in the input dataset."
+ throw new SparkException(msg)
+ }
+
+ val w = if ($(weightCol).isEmpty) lit(1.0) else col($(weightCol))
+ val instances: RDD[Instance] = dataset.select(
+ col($(labelCol)), w, col($(featuresCol))).map {
+ case Row(label: Double, weight: Double, features: Vector) =>
+ Instance(label, weight, features)
+ }
+
+ if ($(family) == "gaussian" && $(link) == "identity") {
+ val optimizer = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true)
--- End diff --
Add a TODO to make those options configurable.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r54524989
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,577 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionBase extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * Default is "gaussian".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "The name of family which is a description of the error distribution to be used in the " +
+ "model. Supported options: gaussian(default), binomial, poisson and gamma.",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilyNames.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of link function which provides the relationship
+ * between the linear predictor and the mean of the distribution function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "The name of link function " +
+ "which provides the relationship between the linear predictor and the mean of the " +
+ "distribution function. Supported options: identity, log, inverse, logit, probit, " +
+ "cloglog and sqrt.",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinkNames.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ import GeneralizedLinearRegression._
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ if ($(solver) == "irls") {
+ setDefault(maxIter -> 25)
+ }
+ if (isDefined(link)) {
+ require(supportedFamilyAndLinkPairs.contains(
+ Family.fromName($(family)) -> Link.fromName($(link))), "Generalized Linear Regression " +
+ s"with ${$(family)} family does not support ${$(link)} link function.")
+ }
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor (link function) and
+ * a description of the error distribution (family).
+ * It supports "gaussian", "binomial", "poisson" and "gamma" as family.
+ * Valid link functions for each family is listed below. The first link function of each family
+ * is the default one.
+ * - "gaussian" -> "identity", "log", "inverse"
+ * - "binomial" -> "logit", "probit", "cloglog"
+ * - "poisson" -> "log", "identity", "sqrt"
+ * - "gamma" -> "inverse", "identity", "log"
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionBase with Logging {
+
+ import GeneralizedLinearRegression._
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("glm"))
+
+ /**
+ * Sets the value of param [[family]].
+ * Default is "gaussian".
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+ setDefault(family -> Gaussian.name)
+
+ /**
+ * Sets the value of param [[link]].
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Sets if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+
+ /**
+ * Sets the maximum number of iterations.
+ * Default is 25 if the solver algorithm is "irls".
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+
+ /**
+ * Sets the convergence tolerance of iterations.
+ * Smaller value will lead to higher accuracy with the cost of more iterations.
+ * Default is 1E-6.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setTol(value: Double): this.type = set(tol, value)
+ setDefault(tol -> 1E-6)
+
+ /**
+ * Sets the regularization parameter.
+ * Default is 0.0.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setRegParam(value: Double): this.type = set(regParam, value)
+ setDefault(regParam -> 0.0)
+
+ /**
+ * Sets the value of param [[weightCol]].
+ * If this is not set or empty, we treat all instance weights as 1.0.
+ * Default is empty, so all instances have weight one.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setWeightCol(value: String): this.type = set(weightCol, value)
+ setDefault(weightCol -> "")
+
+ /**
+ * Sets the solver algorithm used for optimization.
+ * Currently only support "irls" which is also the default solver.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setSolver(value: String): this.type = set(solver, value)
+ setDefault(solver -> "irls")
+
+ override protected def train(dataset: DataFrame): GeneralizedLinearRegressionModel = {
+ val familyObj = Family.fromName($(family))
+ val linkObj = if (isDefined(link)) {
+ Link.fromName($(link))
+ } else {
+ familyObj.defaultLink
+ }
+ val familyAndLink = new FamilyAndLink(familyObj, linkObj)
+
+ val numFeatures = dataset.select(col($(featuresCol))).limit(1)
+ .map { case Row(features: Vector) =>
+ features.size
+ }.first()
+ if (numFeatures > WeightedLeastSquares.MAX_NUM_FEATURES) {
+ val msg = "Currently, GeneralizedLinearRegression only supports number of features" +
+ s" <= ${WeightedLeastSquares.MAX_NUM_FEATURES}. Found $numFeatures in the input dataset."
+ throw new SparkException(msg)
+ }
+
+ val w = if ($(weightCol).isEmpty) lit(1.0) else col($(weightCol))
+ val instances: RDD[Instance] = dataset.select(col($(labelCol)), w, col($(featuresCol)))
+ .map { case Row(label: Double, weight: Double, features: Vector) =>
--- End diff --
`.rdd.map` instead of `.map`. This is caused by recent DataFrame API changes.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by yanboliang <gi...@git.apache.org>.
Github user yanboliang commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r53909986
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,565 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionBase extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * Default is "gaussian".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "The name of family which is a description of the error distribution to be used in the " +
+ "model. Supported options: gaussian(default), binomial, poisson and gamma.",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilyNames.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of link function which provides the relationship
+ * between the linear predictor and the mean of the distribution function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "The name of link function " +
+ "which provides the relationship between the linear predictor and the mean of the " +
+ "distribution function. Supported options: identity, log, inverse, logit, probit, " +
+ "cloglog and sqrt.",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinkNames.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ import GeneralizedLinearRegression._
+ protected lazy val familyObj = Family.fromName($(family))
+ protected lazy val linkObj = if (isDefined(link)) {
+ Link.fromName($(link))
+ } else {
+ familyObj.defaultLink
+ }
+ protected lazy val familyAndLink = new FamilyAndLink(familyObj, linkObj)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ if ($(solver) == "irls") {
+ setDefault(maxIter -> 25)
+ }
+ if (isDefined(link)) {
+ require(GeneralizedLinearRegression.supportedFamilyAndLinkParis.contains(
+ familyObj -> linkObj), s"Generalized Linear Regression with ${$(family)} family " +
+ s"does not support ${$(link)} link function.")
+ }
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor (link function) and
+ * a description of the error distribution (family).
+ * It supports "gaussian", "binomial", "poisson" and "gamma" as family.
+ * Valid link functions for each family is listed below. The first link function of each family
+ * is the default one.
+ * - "gaussian" -> "identity", "log", "inverse"
+ * - "binomial" -> "logit", "probit", "cloglog"
+ * - "poisson" -> "log", "identity", "sqrt"
+ * - "gamma" -> "inverse", "identity", "log"
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionBase with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("glm"))
+
+ /**
+ * Sets the value of param [[family]].
+ * Default is "gaussian".
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+ setDefault(family -> "gaussian")
+
+ /**
+ * Sets the value of param [[link]].
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Sets if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+
+ /**
+ * Sets the maximum number of iterations.
+ * Default is 25 if the solver algorithm is "irls".
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
--- End diff --
We might need to assign different default values for IRLS and L-BFGS, so I did not set default value here.
It means the default value of ```maxIter``` depends on another parameter ```solver```, so I call ```setDefault(maxIter -> 25)``` at [```validateParams```](https://github.com/apache/spark/pull/11136/files#diff-6759d92c079f0957bfa814e339e10e7eR88).
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-188725749
**[Test build #51963 has started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/51963/consoleFull)** for PR 11136 at commit [`2ebcef7`](https://github.com/apache/spark/commit/2ebcef728315ad6b03d48c7f7e7f504e5e193748).
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r53722631
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,547 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ if (isDefined(link)) {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
+ s"Generalized Linear Regression with ${$(family)} family does not support ${$(link)} " +
+ s"link function.")
+ }
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor and
+ * a description of the error distribution.
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("genLinReg"))
+
+ /**
+ * Set the name of family which is a description of the error distribution
+ * to be used in the model.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+
+ /**
+ * Set the name of the model link function.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Set if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+ setDefault(fitIntercept -> true)
+
+ /**
+ * Set the maximum number of iterations.
+ * Default is 100.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+ setDefault(maxIter -> 100)
+
+ /**
+ * Set the convergence tolerance of iterations.
+ * Smaller value will lead to higher accuracy with the cost of more iterations.
+ * Default is 1E-6.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setTol(value: Double): this.type = set(tol, value)
+ setDefault(tol -> 1E-6)
+
+ /**
+ * Set the regularization parameter.
+ * Default is 0.0.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setRegParam(value: Double): this.type = set(regParam, value)
+ setDefault(regParam -> 0.0)
+
+ /**
+ * Whether to over-/under-sample training instances according to the given weights in weightCol.
+ * If empty, all instances are treated equally (weight 1.0).
+ * Default is empty, so all instances have weight one.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setWeightCol(value: String): this.type = set(weightCol, value)
+ setDefault(weightCol -> "")
+
+ /**
+ * Set the solver algorithm used for optimization.
+ * Currently only support "irls" which is also the default solver.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setSolver(value: String): this.type = set(solver, value)
+ setDefault(solver -> "irls")
+
+ override protected def train(dataset: DataFrame): GeneralizedLinearRegressionModel = {
+ val familyLink = $(family) match {
+ case "gaussian" => if (isDefined(link)) Gaussian($(link)) else Gaussian("identity")
+ case "binomial" => if (isDefined(link)) Binomial($(link)) else Binomial("logit")
+ case "poisson" => if (isDefined(link)) Poisson($(link)) else Poisson("log")
+ case "gamma" => if (isDefined(link)) Gamma($(link)) else Gamma("inverse")
+ }
+
+ val numFeatures = dataset.select(col($(featuresCol))).limit(1).map {
+ case Row(features: Vector) => features.size
+ }.first()
+ if (numFeatures > 4096) {
+ val msg = "Currently, GeneralizedLinearRegression only supports number of features" +
+ s" <= 4096. Found $numFeatures in the input dataset."
+ throw new SparkException(msg)
+ }
+
+ val w = if ($(weightCol).isEmpty) lit(1.0) else col($(weightCol))
+ val instances: RDD[Instance] = dataset.select(
+ col($(labelCol)), w, col($(featuresCol))).map {
+ case Row(label: Double, weight: Double, features: Vector) =>
+ Instance(label, weight, features)
+ }
+
+ if ($(family) == "gaussian" && $(link) == "identity") {
+ val optimizer = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true)
+ val wlsModel = optimizer.fit(instances)
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ wlsModel.coefficients, wlsModel.intercept).setParent(this))
+ return model
+ }
+
+ val newInstances = instances.map { instance =>
+ val mu = familyLink.initialize(instance.label, instance.weight)
+ val eta = familyLink.predict(mu)
+ Instance(eta, instance.weight, instance.features)
+ }
+
+ val initialModel = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true).fit(newInstances)
+
+ val reweightFunc: (Instance, WeightedLeastSquaresModel) => (Double, Double) = {
--- End diff --
document the result type
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by AmplabJenkins <gi...@git.apache.org>.
Github user AmplabJenkins commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-190527134
Merged build finished. Test FAILed.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-188129136
**[Test build #51857 has started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/51857/consoleFull)** for PR 11136 at commit [`aa89fdc`](https://github.com/apache/spark/commit/aa89fdcb837e12481bed23f781925ea6e8f6acbe).
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by yanboliang <gi...@git.apache.org>.
Github user yanboliang commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r52882530
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,472 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.Logging
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
+ s"Generalized Linear Regression with ${$(family)} family does not support ${$(link)} " +
+ s"link function.")
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor and
+ * a description of the error distribution.
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("genLinReg"))
+
+ /**
+ * Set the name of family which is a description of the error distribution
+ * to be used in the model.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+
+ /**
+ * Set the name of the model link function.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Set if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+ setDefault(fitIntercept -> true)
+
+ /**
+ * Set the maximum number of iterations.
+ * Default is 100.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+ setDefault(maxIter -> 100)
+
+ /**
+ * Set the convergence tolerance of iterations.
+ * Smaller value will lead to higher accuracy with the cost of more iterations.
+ * Default is 1E-6.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setTol(value: Double): this.type = set(tol, value)
+ setDefault(tol -> 1E-6)
+
+ /**
+ * Set the regularization parameter.
+ * Default is 0.0.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setRegParam(value: Double): this.type = set(regParam, value)
+ setDefault(regParam -> 0.0)
+
+ /**
+ * Whether to over-/under-sample training instances according to the given weights in weightCol.
+ * If empty, all instances are treated equally (weight 1.0).
+ * Default is empty, so all instances have weight one.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setWeightCol(value: String): this.type = set(weightCol, value)
+ setDefault(weightCol -> "")
+
+ /**
+ * Set the solver algorithm used for optimization.
+ * Currently only support "irls" which is also the default solver.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setSolver(value: String): this.type = set(solver, value)
+ setDefault(solver -> "irls")
+
+ override protected def train(dataset: DataFrame): GeneralizedLinearRegressionModel = {
+ val familyLink = $(family) match {
+ case "gaussian" => Gaussian($(link))
+ case "binomial" => Binomial($(link))
+ case "poisson" => Poisson($(link))
+ case "gamma" => Gamma($(link))
+ }
+
+ val w = if ($(weightCol).isEmpty) lit(1.0) else col($(weightCol))
+ val instances: RDD[Instance] = dataset.select(
+ col($(labelCol)), w, col($(featuresCol))).map {
+ case Row(label: Double, weight: Double, features: Vector) =>
+ Instance(label, weight, features)
+ }
+
+ if ($(family) == "gaussian" && $(link) == "identity") {
+ val optimizer = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true)
+ val wlsModel = optimizer.fit(instances)
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ wlsModel.coefficients, wlsModel.intercept).setParent(this))
+ return model
+ }
+
+ val newInstances = instances.map { instance =>
+ val mu = familyLink.initialize(instance.label, instance.weight)
+ val eta = familyLink.predict(mu)
+ Instance(eta, instance.weight, instance.features)
+ }
+
+ val initialModel = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true).fit(newInstances)
+
+ val reweightFunc: (Instance, WeightedLeastSquaresModel) => (Double, Double) = {
+ (instance: Instance, model: WeightedLeastSquaresModel) => {
+ val eta = model.predict(instance.features)
+ val mu = familyLink.fitted(eta)
+ val z = familyLink.adjusted(instance.label, mu, eta)
+ val w = familyLink.weights(mu) * instance.weight
+ (z, w)
+ }
+ }
+
+ val optimizer = new IterativelyReweightedLeastSquares(initialModel, reweightFunc,
+ $(fitIntercept), $(regParam), $(maxIter), $(tol))
+
+ val irlsModel = optimizer.fit(instances)
+
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ irlsModel.coefficients, irlsModel.intercept).setParent(this))
+ model
+ }
+
+ @Since("2.0.0")
+ override def copy(extra: ParamMap): GeneralizedLinearRegression = defaultCopy(extra)
+}
+
+@Since("2.0.0")
+object GeneralizedLinearRegression {
+
+ /** Set of families that GeneralizedLinearRegression supports */
+ private[ml] val supportedFamilies = Set("gaussian", "binomial", "poisson", "gamma")
+
+ /** Set of links that GeneralizedLinearRegression supports */
+ private[ml] val supportedLinks = Set("identity", "log", "inverse", "logit", "probit",
+ "cloglog", "sqrt")
+
+ /** Set of family and link pairs that GeneralizedLinearRegression supports */
+ private[ml] val supportedFamilyLinkPairs = Set(
+ "gaussian" -> "identity", "gaussian" -> "log", "gaussian" -> "inverse",
+ "binomial" -> "logit", "binomial" -> "probit", "binomial" -> "cloglog",
+ "poisson" -> "log", "poisson" -> "identity", "poisson" -> "sqrt",
+ "gamma" -> "inverse", "gamma" -> "identity", "gamma" -> "log"
+ )
+}
+
+/**
+ * :: Experimental ::
+ * Model produced by [[GeneralizedLinearRegression]].
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegressionModel private[ml] (
+ @Since("2.0.0") override val uid: String,
+ @Since("2.0.0") val coefficients: Vector,
+ @Since("2.0.0") val intercept: Double)
+ extends RegressionModel[Vector, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams {
+
+ private lazy val familyLink = $(family) match {
+ case "gaussian" => Gaussian($(link))
+ case "binomial" => Binomial($(link))
+ case "poisson" => Poisson($(link))
+ case "gamma" => Gamma($(link))
+ }
+
+ override protected def predict(features: Vector): Double = {
+ val eta = BLAS.dot(features, coefficients) + intercept
+ familyLink.fitted(eta)
+ }
+
+ @Since("2.0.0")
+ override def copy(extra: ParamMap): GeneralizedLinearRegressionModel = {
+ copyValues(new GeneralizedLinearRegressionModel(uid, coefficients, intercept), extra)
+ .setParent(parent)
+ }
+}
+
+/**
+ * A description of the error distribution and link function to be used in the model.
+ * @param link a link function instance
+ */
+private[ml] abstract class Family(val link: Link) extends Serializable {
+
+ /** Initialize the starting value for mu. */
+ def initialize(y: Double, weight: Double): Double
+
+ /** The variance of mu to its mean. */
+ def variance(mu: Double): Double
+
+ /** Weights for IRLS steps. */
+ def weights(mu: Double): Double = {
+ 1.0 / (math.pow(this.link.deriv(mu), 2.0) * this.variance(mu))
+ }
+
+ /** The adjusted response variable. */
+ def adjusted(y: Double, mu: Double, eta: Double): Double = {
+ eta + (y - mu) * link.deriv(mu)
+ }
+
+ /** Linear predictors based on given mu. */
+ def predict(mu: Double): Double = this.link.link(mu)
+
+ /** Fitted values based on linear predictors eta. */
+ def fitted(eta: Double): Double = this.link.unlink(eta)
+}
+
+/**
+ * Gaussian exponential family distribution.
+ * The default link for the Gaussian family is the identity link.
+ * @param link a link function instance
+ */
+private[ml] class Gaussian(link: Link = Identity) extends Family(link) {
+
+ override def initialize(y: Double, weight: Double): Double = y
+
+ def variance(mu: Double): Double = 1.0
+}
+
+private[ml] object Gaussian {
+
+ def apply(link: String): Gaussian = {
+ link match {
+ case "identity" => new Gaussian(Identity)
+ case "log" => new Gaussian(Log)
+ case "inverse" => new Gaussian(Inverse)
+ }
+ }
+}
+
+/**
+ * Binomial exponential family distribution.
+ * The default link for the Binomial family is the logit link.
+ * @param link a link function instance
+ */
+private[ml] class Binomial(link: Link = Logit) extends Family(link) {
+
+ override def initialize(y: Double, weight: Double): Double = {
+ (weight * y + 0.5) / (weight + 1.0)
+ }
+
+ override def variance(mu: Double): Double = mu * (1 - mu)
+}
+
+private[ml] object Binomial {
+
+ def apply(link: String): Binomial = {
+ link match {
+ case "logit" => new Binomial(Logit)
+ case "probit" => new Binomial(Probit)
+ case "cloglog" => new Binomial(CLogLog)
+ }
+ }
+}
+
+/**
+ * Poisson exponential family distribution.
+ * The default link for the Poisson family is the log link.
+ * @param link a link function instance
+ */
+private[ml] class Poisson(link: Link = Log) extends Family(link) {
+
+ override def initialize(y: Double, weight: Double): Double = y + 0.1
+
+ override def variance(mu: Double): Double = mu
+}
+
+private[ml] object Poisson {
+
+ def apply(link: String): Poisson = {
+ link match {
+ case "log" => new Poisson(Log)
+ case "sqrt" => new Poisson(Sqrt)
+ case "identity" => new Poisson(Identity)
+ }
+ }
+}
+
+/**
+ * Gamma exponential family distribution.
+ * The default link for the Gamma family is the inverse link.
+ * @param link a link function instance
+ */
+private[ml] class Gamma(link: Link = Log) extends Family(link) {
+
+ override def initialize(y: Double, weight: Double): Double = y
+
+ override def variance(mu: Double): Double = math.pow(mu, 2.0)
+}
+
+private[ml] object Gamma {
+
+ def apply(link: String): Gamma = {
+ link match {
+ case "inverse" => new Gamma(Inverse)
+ case "identity" => new Gamma(Identity)
+ case "log" => new Gamma(Log)
+ }
+ }
+}
+
+/**
+ * A description of the link function to be used in the model.
+ */
+private[ml] trait Link extends Serializable {
+
+ /** The link function. */
+ def link(mu: Double): Double
+
+ /** The derivative function. */
+ def deriv(mu: Double): Double
+
+ /** The inverse link function. */
+ def unlink(eta: Double): Double
+}
+
+private[ml] object Identity extends Link {
+
+ override def link(mu: Double): Double = mu
+
+ override def deriv(mu: Double): Double = 1.0
+
+ override def unlink(eta: Double): Double = eta
+}
+
+private[ml] object Logit extends Link {
+
+ override def link(mu: Double): Double = math.log(mu / (1.0 - mu))
+
+ override def deriv(mu: Double): Double = 1.0 / (mu * (1.0 - mu))
+
+ override def unlink(eta: Double): Double = 1.0 / (1.0 + math.exp(-1.0 * eta))
+}
--- End diff --
Yes, I have add restriction in ```Family```.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-190522453
**[Test build #52211 has finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/52211/consoleFull)** for PR 11136 at commit [`314b562`](https://github.com/apache/spark/commit/314b562f315723a7117851289c8f5b6e1b16a6ac).
* This patch **fails to build**.
* This patch merges cleanly.
* This patch adds no public classes.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by AmplabJenkins <gi...@git.apache.org>.
Github user AmplabJenkins commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-183874429
Test PASSed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/51264/
Test PASSed.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r53722508
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,547 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ if (isDefined(link)) {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
+ s"Generalized Linear Regression with ${$(family)} family does not support ${$(link)} " +
+ s"link function.")
+ }
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor and
+ * a description of the error distribution.
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("genLinReg"))
+
+ /**
+ * Set the name of family which is a description of the error distribution
+ * to be used in the model.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+
+ /**
+ * Set the name of the model link function.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Set if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+ setDefault(fitIntercept -> true)
+
+ /**
+ * Set the maximum number of iterations.
+ * Default is 100.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+ setDefault(maxIter -> 100)
+
+ /**
+ * Set the convergence tolerance of iterations.
+ * Smaller value will lead to higher accuracy with the cost of more iterations.
+ * Default is 1E-6.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setTol(value: Double): this.type = set(tol, value)
+ setDefault(tol -> 1E-6)
+
+ /**
+ * Set the regularization parameter.
+ * Default is 0.0.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setRegParam(value: Double): this.type = set(regParam, value)
+ setDefault(regParam -> 0.0)
+
+ /**
+ * Whether to over-/under-sample training instances according to the given weights in weightCol.
+ * If empty, all instances are treated equally (weight 1.0).
+ * Default is empty, so all instances have weight one.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setWeightCol(value: String): this.type = set(weightCol, value)
+ setDefault(weightCol -> "")
+
+ /**
+ * Set the solver algorithm used for optimization.
+ * Currently only support "irls" which is also the default solver.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setSolver(value: String): this.type = set(solver, value)
+ setDefault(solver -> "irls")
+
+ override protected def train(dataset: DataFrame): GeneralizedLinearRegressionModel = {
+ val familyLink = $(family) match {
+ case "gaussian" => if (isDefined(link)) Gaussian($(link)) else Gaussian("identity")
+ case "binomial" => if (isDefined(link)) Binomial($(link)) else Binomial("logit")
+ case "poisson" => if (isDefined(link)) Poisson($(link)) else Poisson("log")
+ case "gamma" => if (isDefined(link)) Gamma($(link)) else Gamma("inverse")
+ }
+
+ val numFeatures = dataset.select(col($(featuresCol))).limit(1).map {
+ case Row(features: Vector) => features.size
+ }.first()
+ if (numFeatures > 4096) {
+ val msg = "Currently, GeneralizedLinearRegression only supports number of features" +
+ s" <= 4096. Found $numFeatures in the input dataset."
+ throw new SparkException(msg)
+ }
+
+ val w = if ($(weightCol).isEmpty) lit(1.0) else col($(weightCol))
+ val instances: RDD[Instance] = dataset.select(
+ col($(labelCol)), w, col($(featuresCol))).map {
--- End diff --
fix indentation for readability, e.g,:
~~~scala
val instances: RDD[Instance] = dataset.select(col($(labelCol)), w, col($(featuresCol)))
.map { case Row(label: Double, weight: Double, features: Vector) =>
Instance(label, weight, features)
}
}
~~~
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by yanboliang <gi...@git.apache.org>.
Github user yanboliang commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r54206474
--- Diff: mllib/src/main/scala/org/apache/spark/ml/optim/WeightedLeastSquares.scala ---
@@ -157,6 +157,12 @@ private[ml] class WeightedLeastSquares(
private[ml] object WeightedLeastSquares {
/**
+ * In order to take the normal equation approach efficiently, [[WeightedLeastSquares]]
+ * only supports the number of features is no more than 4096.
+ */
+ val MaxNumFeatures: Int = 4096
--- End diff --
OK, I will update it to ```MAX_NUM_FEATURES``` after collecting more comments. Thanks!
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-181963578
**[Test build #50980 has finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/50980/consoleFull)** for PR 11136 at commit [`3082686`](https://github.com/apache/spark/commit/3082686d61a851c240f16a9cd7aafc97fa6e3296).
* This patch passes all tests.
* This patch merges cleanly.
* This patch adds no public classes.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by sethah <gi...@git.apache.org>.
Github user sethah commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r52405040
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,472 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.Logging
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
+ s"Generalized Linear Regression with ${$(family)} family does not support ${$(link)} " +
+ s"link function.")
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor and
+ * a description of the error distribution.
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("genLinReg"))
+
+ /**
+ * Set the name of family which is a description of the error distribution
+ * to be used in the model.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+
+ /**
+ * Set the name of the model link function.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Set if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+ setDefault(fitIntercept -> true)
+
+ /**
+ * Set the maximum number of iterations.
+ * Default is 100.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+ setDefault(maxIter -> 100)
+
+ /**
+ * Set the convergence tolerance of iterations.
+ * Smaller value will lead to higher accuracy with the cost of more iterations.
+ * Default is 1E-6.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setTol(value: Double): this.type = set(tol, value)
+ setDefault(tol -> 1E-6)
+
+ /**
+ * Set the regularization parameter.
+ * Default is 0.0.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setRegParam(value: Double): this.type = set(regParam, value)
+ setDefault(regParam -> 0.0)
+
+ /**
+ * Whether to over-/under-sample training instances according to the given weights in weightCol.
+ * If empty, all instances are treated equally (weight 1.0).
+ * Default is empty, so all instances have weight one.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setWeightCol(value: String): this.type = set(weightCol, value)
+ setDefault(weightCol -> "")
+
+ /**
+ * Set the solver algorithm used for optimization.
+ * Currently only support "irls" which is also the default solver.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setSolver(value: String): this.type = set(solver, value)
+ setDefault(solver -> "irls")
+
+ override protected def train(dataset: DataFrame): GeneralizedLinearRegressionModel = {
--- End diff --
When `train` is called with a dataframe that has more than 4096 features, you get the error:
"org.apache.spark.SparkException: Job aborted due to stage failure: Task 1 in stage 5.0 failed 1 times, most recent failure: Lost task 1.0 in stage 5.0 (TID 6, localhost): java.lang.IllegalArgumentException: requirement failed: In order to take the normal equation approach efficiently, we set the max number of features to 4096 but got 5000."
Which is _ok_ but IMO not ideal. It may not be clear to the user where exactly the normal equation approach is happening. We could check the number of features before calling the WLS algorithm and throw a clearer exception. Additionally, should we state somewhere in the API docs that GLMs cannot handle more than 4096 features?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r54059572
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,565 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionBase extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * Default is "gaussian".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "The name of family which is a description of the error distribution to be used in the " +
+ "model. Supported options: gaussian(default), binomial, poisson and gamma.",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilyNames.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of link function which provides the relationship
+ * between the linear predictor and the mean of the distribution function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "The name of link function " +
+ "which provides the relationship between the linear predictor and the mean of the " +
+ "distribution function. Supported options: identity, log, inverse, logit, probit, " +
+ "cloglog and sqrt.",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinkNames.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ import GeneralizedLinearRegression._
+ protected lazy val familyObj = Family.fromName($(family))
+ protected lazy val linkObj = if (isDefined(link)) {
+ Link.fromName($(link))
+ } else {
+ familyObj.defaultLink
+ }
+ protected lazy val familyAndLink = new FamilyAndLink(familyObj, linkObj)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ if ($(solver) == "irls") {
+ setDefault(maxIter -> 25)
+ }
+ if (isDefined(link)) {
+ require(GeneralizedLinearRegression.supportedFamilyAndLinkParis.contains(
+ familyObj -> linkObj), s"Generalized Linear Regression with ${$(family)} family " +
+ s"does not support ${$(link)} link function.")
+ }
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor (link function) and
+ * a description of the error distribution (family).
+ * It supports "gaussian", "binomial", "poisson" and "gamma" as family.
+ * Valid link functions for each family is listed below. The first link function of each family
+ * is the default one.
+ * - "gaussian" -> "identity", "log", "inverse"
+ * - "binomial" -> "logit", "probit", "cloglog"
+ * - "poisson" -> "log", "identity", "sqrt"
+ * - "gamma" -> "inverse", "identity", "log"
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionBase with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("glm"))
+
+ /**
+ * Sets the value of param [[family]].
+ * Default is "gaussian".
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+ setDefault(family -> "gaussian")
+
+ /**
+ * Sets the value of param [[link]].
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Sets if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+
+ /**
+ * Sets the maximum number of iterations.
+ * Default is 25 if the solver algorithm is "irls".
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+
+ /**
+ * Sets the convergence tolerance of iterations.
+ * Smaller value will lead to higher accuracy with the cost of more iterations.
+ * Default is 1E-6.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setTol(value: Double): this.type = set(tol, value)
+ setDefault(tol -> 1E-6)
+
+ /**
+ * Sets the regularization parameter.
+ * Default is 0.0.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setRegParam(value: Double): this.type = set(regParam, value)
+ setDefault(regParam -> 0.0)
+
+ /**
+ * Sets the value of param [[weightCol]].
+ * If this is not set or empty, we treat all instance weights as 1.0.
+ * Default is empty, so all instances have weight one.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setWeightCol(value: String): this.type = set(weightCol, value)
+ setDefault(weightCol -> "")
+
+ /**
+ * Sets the solver algorithm used for optimization.
+ * Currently only support "irls" which is also the default solver.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setSolver(value: String): this.type = set(solver, value)
+ setDefault(solver -> "irls")
+
+ override protected def train(dataset: DataFrame): GeneralizedLinearRegressionModel = {
+ val numFeatures = dataset.select(col($(featuresCol))).limit(1)
+ .map { case Row(features: Vector) =>
+ features.size
+ }.first()
+ if (numFeatures > WeightedLeastSquares.MaxNumFeatures) {
+ val msg = "Currently, GeneralizedLinearRegression only supports number of features" +
+ s" <= ${WeightedLeastSquares.MaxNumFeatures}. Found $numFeatures in the input dataset."
+ throw new SparkException(msg)
+ }
+
+ val w = if ($(weightCol).isEmpty) lit(1.0) else col($(weightCol))
+ val instances: RDD[Instance] = dataset.select(col($(labelCol)), w, col($(featuresCol)))
+ .map { case Row(label: Double, weight: Double, features: Vector) =>
+ Instance(label, weight, features)
+ }
+
+ if (familyObj == GeneralizedLinearRegression.Gaussian &&
+ linkObj == GeneralizedLinearRegression.Identity) {
+ // TODO: Make standardizeFeatures and standardizeLabel configurable.
+ val optimizer = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true)
+ val wlsModel = optimizer.fit(instances)
+ val model = copyValues(
+ new GeneralizedLinearRegressionModel(uid, wlsModel.coefficients, wlsModel.intercept)
+ .setParent(this))
+ return model
+ }
+
+ // Fit Generalized Linear Model by iteratively reweighted least squares (IRLS).
+ val initialModel = familyAndLink.initialize(instances, $(fitIntercept), $(regParam))
+ val optimizer = new IterativelyReweightedLeastSquares(initialModel, familyAndLink.reweightFunc,
+ $(fitIntercept), $(regParam), $(maxIter), $(tol))
+ val irlsModel = optimizer.fit(instances)
+
+ val model = copyValues(
+ new GeneralizedLinearRegressionModel(uid, irlsModel.coefficients, irlsModel.intercept)
+ .setParent(this))
+ model
+ }
+
+ @Since("2.0.0")
+ override def copy(extra: ParamMap): GeneralizedLinearRegression = defaultCopy(extra)
+}
+
+@Since("2.0.0")
+private[ml] object GeneralizedLinearRegression {
+
+ /** Set of family and link pairs that GeneralizedLinearRegression supports. */
+ lazy val supportedFamilyAndLinkParis = Set(
--- End diff --
`Pairs`
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-188138376
**[Test build #51857 has finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/51857/consoleFull)** for PR 11136 at commit [`aa89fdc`](https://github.com/apache/spark/commit/aa89fdcb837e12481bed23f781925ea6e8f6acbe).
* This patch passes all tests.
* This patch merges cleanly.
* This patch adds no public classes.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-190806584
LGTM. Merged into master. Thanks!
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by asfgit <gi...@git.apache.org>.
Github user asfgit closed the pull request at:
https://github.com/apache/spark/pull/11136
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by AmplabJenkins <gi...@git.apache.org>.
Github user AmplabJenkins commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-190530642
Test FAILed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/52215/
Test FAILed.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by AmplabJenkins <gi...@git.apache.org>.
Github user AmplabJenkins commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-181964091
Test PASSed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/50980/
Test PASSed.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-190591444
**[Test build #52227 has started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/52227/consoleFull)** for PR 11136 at commit [`007a4ec`](https://github.com/apache/spark/commit/007a4ec324db273c048ed65fe8942daba0c9d844).
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by AmplabJenkins <gi...@git.apache.org>.
Github user AmplabJenkins commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-190607485
Test PASSed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/52227/
Test PASSed.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by yanboliang <gi...@git.apache.org>.
Github user yanboliang commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r54521794
--- Diff: mllib/src/test/scala/org/apache/spark/ml/regression/GeneralizedLinearRegressionSuite.scala ---
@@ -0,0 +1,499 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import scala.util.Random
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.ml.param.ParamsSuite
+import org.apache.spark.ml.util.MLTestingUtils
+import org.apache.spark.mllib.classification.LogisticRegressionSuite._
+import org.apache.spark.mllib.linalg.{BLAS, DenseVector, Vectors}
+import org.apache.spark.mllib.random._
+import org.apache.spark.mllib.regression.LabeledPoint
+import org.apache.spark.mllib.util.MLlibTestSparkContext
+import org.apache.spark.mllib.util.TestingUtils._
+import org.apache.spark.sql.{DataFrame, Row}
+
+class GeneralizedLinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
+
+ private val seed: Int = 42
+ @transient var datasetGaussianIdentity: DataFrame = _
+ @transient var datasetGaussianLog: DataFrame = _
+ @transient var datasetGaussianInverse: DataFrame = _
+ @transient var datasetBinomial: DataFrame = _
+ @transient var datasetPoissonLog: DataFrame = _
+ @transient var datasetPoissonIdentity: DataFrame = _
+ @transient var datasetPoissonSqrt: DataFrame = _
+ @transient var datasetGammaInverse: DataFrame = _
+ @transient var datasetGammaIdentity: DataFrame = _
+ @transient var datasetGammaLog: DataFrame = _
+
+ override def beforeAll(): Unit = {
+ super.beforeAll()
+
+ import GeneralizedLinearRegressionSuite._
+
+ datasetGaussianIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "identity"), 2))
+
+ datasetGaussianLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "log"), 2))
+
+ datasetGaussianInverse = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "inverse"), 2))
+
+ datasetBinomial = {
+ val nPoints = 10000
+ val coefficients = Array(-0.57997, 0.912083, -0.371077, -0.819866, 2.688191)
+ val xMean = Array(5.843, 3.057, 3.758, 1.199)
+ val xVariance = Array(0.6856, 0.1899, 3.116, 0.581)
+
+ val testData =
+ generateMultinomialLogisticInput(coefficients, xMean, xVariance, true, nPoints, seed)
+
+ sqlContext.createDataFrame(sc.parallelize(testData, 4))
+ }
+
+ datasetPoissonLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "log"), 2))
+
+ datasetPoissonIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "identity"), 2))
+
+ datasetPoissonSqrt = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "sqrt"), 2))
+
+ datasetGammaInverse = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "inverse"), 2))
+
+ datasetGammaIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "identity"), 2))
+
+ datasetGammaLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "log"), 2))
+ }
+
+ test("params") {
+ ParamsSuite.checkParams(new GeneralizedLinearRegression)
+ val model = new GeneralizedLinearRegressionModel("genLinReg", Vectors.dense(0.0), 0.0)
+ ParamsSuite.checkParams(model)
+ }
+
+ test("generalized linear regression: default params") {
+ val glr = new GeneralizedLinearRegression
+ assert(glr.getLabelCol === "label")
+ assert(glr.getFeaturesCol === "features")
+ assert(glr.getPredictionCol === "prediction")
+ assert(glr.getFitIntercept)
+ assert(glr.getTol === 1E-6)
+ assert(glr.getWeightCol === "")
+ assert(glr.getRegParam === 0.0)
+ assert(glr.getSolver == "irls")
+ val model = glr.setFamily("gaussian").setLink("identity")
+ .fit(datasetGaussianIdentity)
+
+ // copied model must have the same parent.
+ MLTestingUtils.checkCopy(model)
+ model.transform(datasetGaussianIdentity)
+ .select("label", "prediction")
+ .collect()
+
+ assert(model.getFeaturesCol === "features")
+ assert(model.getPredictionCol === "prediction")
+ assert(model.intercept !== 0.0)
+ assert(model.hasParent)
+ assert(model.getFamily === "gaussian")
+ assert(model.getLink === "identity")
+ }
+
+ test("generalized linear regression: gaussian family against glm") {
+ /*
+ R code:
+ f1 <- data$V1 ~ data$V2 + data$V3 - 1
+ f2 <- data$V1 ~ data$V2 + data$V3
+
+ data <- read.csv("path", header=FALSE)
+ for (formula in c(f1, f2)) {
+ model <- glm(formula, family="gaussian", data=data)
+ print(as.vector(coef(model)))
+ }
+
+ [1] 2.2960999 0.8087933
+ [1] 2.5002642 2.2000403 0.5999485
+
+ data <- read.csv("path", header=FALSE)
+ model1 <- glm(f1, family=gaussian(link=log), data=data, start=c(0,0))
+ model2 <- glm(f2, family=gaussian(link=log), data=data, start=c(0,0,0))
+ print(as.vector(coef(model1)))
+ print(as.vector(coef(model2)))
+
+ [1] 0.23069326 0.07993778
+ [1] 0.25001858 0.22002452 0.05998789
+
+ data <- read.csv("path", header=FALSE)
+ for (formula in c(f1, f2)) {
+ model <- glm(formula, family=gaussian(link=inverse), data=data)
+ print(as.vector(coef(model)))
+ }
+
+ [1] 2.3010179 0.8198976
+ [1] 2.4108902 2.2130248 0.6086152
+ */
+
+ val expected = Seq(
+ Vectors.dense(0.0, 2.2960999, 0.8087933),
+ Vectors.dense(2.5002642, 2.2000403, 0.5999485),
+ Vectors.dense(0.0, 0.23069326, 0.07993778),
+ Vectors.dense(0.25001858, 0.22002452, 0.05998789),
+ Vectors.dense(0.0, 2.3010179, 0.8198976),
+ Vectors.dense(2.4108902, 2.2130248, 0.6086152))
+
+ import GeneralizedLinearRegression._
+
+ var idx = 0
+ for ((link, dataset) <- Seq(("identity", datasetGaussianIdentity), ("log", datasetGaussianLog),
+ ("inverse", datasetGaussianInverse))) {
+ for (fitIntercept <- Seq(false, true)) {
+ val trainer = new GeneralizedLinearRegression().setFamily("gaussian").setLink(link)
+ .setFitIntercept(fitIntercept)
+ val model = trainer.fit(dataset)
+ val actual = Vectors.dense(model.intercept, model.coefficients(0), model.coefficients(1))
+ assert(actual ~== expected(idx) absTol 1e-4)
+
+ val familyLink = new FamilyAndLink(Gaussian, Link.fromName(link))
+ model.transform(dataset).select("features", "prediction").collect().foreach {
+ case Row(features: DenseVector, prediction1: Double) =>
+ val eta = BLAS.dot(features, model.coefficients) + model.intercept
+ val prediction2 = familyLink.fitted(eta)
+ assert(prediction1 ~== prediction2 relTol 1E-5)
--- End diff --
Fixed.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by yanboliang <gi...@git.apache.org>.
Github user yanboliang commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-190528993
Jenkins, test this please.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-190520555
**[Test build #52211 has started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/52211/consoleFull)** for PR 11136 at commit [`314b562`](https://github.com/apache/spark/commit/314b562f315723a7117851289c8f5b6e1b16a6ac).
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by yanboliang <gi...@git.apache.org>.
Github user yanboliang commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r52611593
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,472 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.Logging
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
--- End diff --
Good point! But we can not check ```isSet(link)``` in the setter for family, because users may set family before set link and it will produce mistake. We can check ```isSet(link)``` at the start of train.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-181949342
**[Test build #50978 has finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/50978/consoleFull)** for PR 11136 at commit [`5af604e`](https://github.com/apache/spark/commit/5af604e180d92a75b18b30717d6516d5ee8135bd).
* This patch passes all tests.
* This patch merges cleanly.
* This patch adds the following public classes _(experimental)_:
* `class GeneralizedLinearRegression @Since(\"2.0.0\") (@Since(\"2.0.0\") override val uid: String)`
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-182917919
**[Test build #51101 has started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/51101/consoleFull)** for PR 11136 at commit [`97c3f6a`](https://github.com/apache/spark/commit/97c3f6a2b94d6705606dd020f61c5c604b562b4c).
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r53722714
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,547 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ if (isDefined(link)) {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
+ s"Generalized Linear Regression with ${$(family)} family does not support ${$(link)} " +
+ s"link function.")
+ }
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor and
+ * a description of the error distribution.
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("genLinReg"))
+
+ /**
+ * Set the name of family which is a description of the error distribution
+ * to be used in the model.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+
+ /**
+ * Set the name of the model link function.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Set if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+ setDefault(fitIntercept -> true)
+
+ /**
+ * Set the maximum number of iterations.
+ * Default is 100.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+ setDefault(maxIter -> 100)
+
+ /**
+ * Set the convergence tolerance of iterations.
+ * Smaller value will lead to higher accuracy with the cost of more iterations.
+ * Default is 1E-6.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setTol(value: Double): this.type = set(tol, value)
+ setDefault(tol -> 1E-6)
+
+ /**
+ * Set the regularization parameter.
+ * Default is 0.0.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setRegParam(value: Double): this.type = set(regParam, value)
+ setDefault(regParam -> 0.0)
+
+ /**
+ * Whether to over-/under-sample training instances according to the given weights in weightCol.
+ * If empty, all instances are treated equally (weight 1.0).
+ * Default is empty, so all instances have weight one.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setWeightCol(value: String): this.type = set(weightCol, value)
+ setDefault(weightCol -> "")
+
+ /**
+ * Set the solver algorithm used for optimization.
+ * Currently only support "irls" which is also the default solver.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setSolver(value: String): this.type = set(solver, value)
+ setDefault(solver -> "irls")
+
+ override protected def train(dataset: DataFrame): GeneralizedLinearRegressionModel = {
+ val familyLink = $(family) match {
+ case "gaussian" => if (isDefined(link)) Gaussian($(link)) else Gaussian("identity")
+ case "binomial" => if (isDefined(link)) Binomial($(link)) else Binomial("logit")
+ case "poisson" => if (isDefined(link)) Poisson($(link)) else Poisson("log")
+ case "gamma" => if (isDefined(link)) Gamma($(link)) else Gamma("inverse")
+ }
+
+ val numFeatures = dataset.select(col($(featuresCol))).limit(1).map {
+ case Row(features: Vector) => features.size
+ }.first()
+ if (numFeatures > 4096) {
+ val msg = "Currently, GeneralizedLinearRegression only supports number of features" +
+ s" <= 4096. Found $numFeatures in the input dataset."
+ throw new SparkException(msg)
+ }
+
+ val w = if ($(weightCol).isEmpty) lit(1.0) else col($(weightCol))
+ val instances: RDD[Instance] = dataset.select(
+ col($(labelCol)), w, col($(featuresCol))).map {
+ case Row(label: Double, weight: Double, features: Vector) =>
+ Instance(label, weight, features)
+ }
+
+ if ($(family) == "gaussian" && $(link) == "identity") {
+ val optimizer = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true)
+ val wlsModel = optimizer.fit(instances)
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ wlsModel.coefficients, wlsModel.intercept).setParent(this))
+ return model
+ }
+
+ val newInstances = instances.map { instance =>
+ val mu = familyLink.initialize(instance.label, instance.weight)
+ val eta = familyLink.predict(mu)
+ Instance(eta, instance.weight, instance.features)
+ }
+
+ val initialModel = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true).fit(newInstances)
+
+ val reweightFunc: (Instance, WeightedLeastSquaresModel) => (Double, Double) = {
+ (instance: Instance, model: WeightedLeastSquaresModel) => {
+ val eta = model.predict(instance.features)
+ val mu = familyLink.fitted(eta)
+ val z = familyLink.adjusted(instance.label, mu, eta)
+ val w = familyLink.weights(mu) * instance.weight
+ (z, w)
+ }
+ }
+
+ val optimizer = new IterativelyReweightedLeastSquares(initialModel, reweightFunc,
+ $(fitIntercept), $(regParam), $(maxIter), $(tol))
+
+ val irlsModel = optimizer.fit(instances)
+
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ irlsModel.coefficients, irlsModel.intercept).setParent(this))
+ model
+ }
+
+ @Since("2.0.0")
+ override def copy(extra: ParamMap): GeneralizedLinearRegression = defaultCopy(extra)
+}
+
+@Since("2.0.0")
+object GeneralizedLinearRegression {
+
+ /** Set of families that GeneralizedLinearRegression supports */
+ private[ml] val supportedFamilies = Set("gaussian", "binomial", "poisson", "gamma")
+
+ /** Set of links that GeneralizedLinearRegression supports */
+ private[ml] val supportedLinks = Set("identity", "log", "inverse", "logit", "probit",
+ "cloglog", "sqrt")
+
+ /** Set of family and link pairs that GeneralizedLinearRegression supports */
+ private[ml] val supportedFamilyLinkPairs = Set(
+ "gaussian" -> "identity", "gaussian" -> "log", "gaussian" -> "inverse",
+ "binomial" -> "logit", "binomial" -> "probit", "binomial" -> "cloglog",
+ "poisson" -> "log", "poisson" -> "identity", "poisson" -> "sqrt",
+ "gamma" -> "inverse", "gamma" -> "identity", "gamma" -> "log"
+ )
+}
+
+/**
+ * :: Experimental ::
+ * Model produced by [[GeneralizedLinearRegression]].
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegressionModel private[ml] (
+ @Since("2.0.0") override val uid: String,
+ @Since("2.0.0") val coefficients: Vector,
+ @Since("2.0.0") val intercept: Double)
+ extends RegressionModel[Vector, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams {
+
+ private lazy val familyLink = $(family) match {
+ case "gaussian" => if (isDefined(link)) Gaussian($(link)) else Gaussian("identity")
+ case "binomial" => if (isDefined(link)) Binomial($(link)) else Binomial("logit")
+ case "poisson" => if (isDefined(link)) Poisson($(link)) else Poisson("log")
+ case "gamma" => if (isDefined(link)) Gamma($(link)) else Gamma("inverse")
+ }
+
+ override protected def predict(features: Vector): Double = {
+ val eta = BLAS.dot(features, coefficients) + intercept
+ familyLink.fitted(eta)
+ }
+
+ @Since("2.0.0")
+ override def copy(extra: ParamMap): GeneralizedLinearRegressionModel = {
+ copyValues(new GeneralizedLinearRegressionModel(uid, coefficients, intercept), extra)
+ .setParent(parent)
+ }
+}
+
+/**
+ * A description of the error distribution and link function to be used in the model.
+ * @param link a link function instance
+ */
+private[ml] abstract class Family(val link: Link) extends Serializable {
+
+ /** Initialize the starting value for mu. */
+ def initialize(y: Double, weight: Double): Double
+
+ /** The variance of the endogenous variable's mean, given the value mu. */
+ def variance(mu: Double): Double
+
+ /** Weights for IRLS steps. */
+ def weights(mu: Double): Double = {
+ val x = clean(mu)
+ 1.0 / (math.pow(this.link.deriv(x), 2.0) * this.variance(x))
+ }
+
+ /** The adjusted response variable. */
+ def adjusted(y: Double, mu: Double, eta: Double): Double = {
+ val x = clean(mu)
+ eta + (y - x) * link.deriv(x)
+ }
+
+ /** Linear predictor based on given mu. */
+ def predict(mu: Double): Double = this.link.link(clean(mu))
+
+ /** Fitted value based on linear predictor eta. */
+ def fitted(eta: Double): Double = clean(this.link.unlink(eta))
+
+ /** Trim the fitted value so that it will be in valid range. */
+ def clean(mu: Double): Double = mu
+
+ val epsilon: Double = 1E-16
+}
+
+/**
+ * Gaussian exponential family distribution.
+ * The default link for the Gaussian family is the identity link.
+ * @param link a link function instance
+ */
+private[ml] class Gaussian(link: Link = Identity) extends Family(link) {
+
+ override def initialize(y: Double, weight: Double): Double = {
+ if (link == Log) {
+ require(y > 0.0, "The response variable of Gaussian family with Log link " +
+ s"should be positive, but got $y")
+ }
+ y
+ }
+
+ def variance(mu: Double): Double = 1.0
+
+ override def clean(mu: Double): Double = {
+ if (mu.isNegInfinity) {
+ Double.MinValue
+ } else if (mu.isPosInfinity) {
+ Double.MaxValue
+ } else {
+ mu
+ }
+ }
+}
+
+private[ml] object Gaussian {
+
+ def apply(link: String): Gaussian = {
+ link match {
+ case "identity" => new Gaussian(Identity)
+ case "log" => new Gaussian(Log)
+ case "inverse" => new Gaussian(Inverse)
+ }
+ }
+}
+
+/**
+ * Binomial exponential family distribution.
+ * The default link for the Binomial family is the logit link.
+ * @param link a link function instance
+ */
+private[ml] class Binomial(link: Link = Logit) extends Family(link) {
+
+ override def initialize(y: Double, weight: Double): Double = {
+ val mu = (weight * y + 0.5) / (weight + 1.0)
+ require(mu > 0.0 && mu < 1.0, "The response variable of Binomial family" +
+ s"should be in range (0, 1), but got $mu")
+ mu
+ }
+
+ override def variance(mu: Double): Double = mu * (1.0 - mu)
+
+ override def clean(mu: Double): Double = {
+ if (mu < epsilon) {
+ epsilon
+ } else if (mu > 1.0 - epsilon) {
+ 1.0 - epsilon
+ } else {
+ mu
+ }
+ }
+}
+
+private[ml] object Binomial {
+
+ def apply(link: String): Binomial = {
+ link match {
+ case "logit" => new Binomial(Logit)
+ case "probit" => new Binomial(Probit)
+ case "cloglog" => new Binomial(CLogLog)
+ }
+ }
+}
+
+/**
+ * Poisson exponential family distribution.
+ * The default link for the Poisson family is the log link.
+ * @param link a link function instance
+ */
+private[ml] class Poisson(link: Link = Log) extends Family(link) {
+
+ override def initialize(y: Double, weight: Double): Double = {
+ require(y > 0.0, "The response variable of Poisson family " +
+ s"should be positive, but got $y")
+ y
+ }
+
+ override def variance(mu: Double): Double = mu
+
+ override def clean(mu: Double): Double = {
+ if (mu < epsilon) {
+ epsilon
+ } else if (mu.isInfinity) {
+ Double.MaxValue
+ } else {
+ mu
+ }
+ }
+}
+
+private[ml] object Poisson {
+
+ def apply(link: String): Poisson = {
+ link match {
+ case "log" => new Poisson(Log)
+ case "sqrt" => new Poisson(Sqrt)
+ case "identity" => new Poisson(Identity)
+ }
+ }
+}
+
+/**
+ * Gamma exponential family distribution.
+ * The default link for the Gamma family is the inverse link.
+ * @param link a link function instance
+ */
+private[ml] class Gamma(link: Link = Inverse) extends Family(link) {
+
+ override def initialize(y: Double, weight: Double): Double = {
+ require(y > 0.0, "The response variable of Gamma family " +
+ s"should be positive, but got $y")
+ y
+ }
+
+ override def variance(mu: Double): Double = math.pow(mu, 2.0)
+
+ override def clean(mu: Double): Double = {
+ if (mu < epsilon) {
+ epsilon
+ } else if (mu.isInfinity) {
+ Double.MaxValue
+ } else {
+ mu
+ }
+ }
+}
+
+private[ml] object Gamma {
+
+ def apply(link: String): Gamma = {
+ link match {
+ case "inverse" => new Gamma(Inverse)
+ case "identity" => new Gamma(Identity)
+ case "log" => new Gamma(Log)
+ }
+ }
+}
+
+/**
+ * A description of the link function to be used in the model.
+ */
+private[ml] trait Link extends Serializable {
+
+ /** The link function. */
+ def link(mu: Double): Double
+
+ /** Derivative of the link function. */
+ def deriv(mu: Double): Double
+
+ /** The inverse link function. */
+ def unlink(eta: Double): Double
+}
--- End diff --
* We can add `name` to `Link` for easy debugging.
* We can add `Link.fromName(name: String): Link` to handle raw strings.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by AmplabJenkins <gi...@git.apache.org>.
Github user AmplabJenkins commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-189211004
Test PASSed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/52044/
Test PASSed.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by AmplabJenkins <gi...@git.apache.org>.
Github user AmplabJenkins commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-184175086
Merged build finished. Test PASSed.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r53722697
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,547 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ if (isDefined(link)) {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
+ s"Generalized Linear Regression with ${$(family)} family does not support ${$(link)} " +
+ s"link function.")
+ }
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor and
+ * a description of the error distribution.
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("genLinReg"))
+
+ /**
+ * Set the name of family which is a description of the error distribution
+ * to be used in the model.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+
+ /**
+ * Set the name of the model link function.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Set if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+ setDefault(fitIntercept -> true)
+
+ /**
+ * Set the maximum number of iterations.
+ * Default is 100.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+ setDefault(maxIter -> 100)
+
+ /**
+ * Set the convergence tolerance of iterations.
+ * Smaller value will lead to higher accuracy with the cost of more iterations.
+ * Default is 1E-6.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setTol(value: Double): this.type = set(tol, value)
+ setDefault(tol -> 1E-6)
+
+ /**
+ * Set the regularization parameter.
+ * Default is 0.0.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setRegParam(value: Double): this.type = set(regParam, value)
+ setDefault(regParam -> 0.0)
+
+ /**
+ * Whether to over-/under-sample training instances according to the given weights in weightCol.
+ * If empty, all instances are treated equally (weight 1.0).
+ * Default is empty, so all instances have weight one.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setWeightCol(value: String): this.type = set(weightCol, value)
+ setDefault(weightCol -> "")
+
+ /**
+ * Set the solver algorithm used for optimization.
+ * Currently only support "irls" which is also the default solver.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setSolver(value: String): this.type = set(solver, value)
+ setDefault(solver -> "irls")
+
+ override protected def train(dataset: DataFrame): GeneralizedLinearRegressionModel = {
+ val familyLink = $(family) match {
+ case "gaussian" => if (isDefined(link)) Gaussian($(link)) else Gaussian("identity")
+ case "binomial" => if (isDefined(link)) Binomial($(link)) else Binomial("logit")
+ case "poisson" => if (isDefined(link)) Poisson($(link)) else Poisson("log")
+ case "gamma" => if (isDefined(link)) Gamma($(link)) else Gamma("inverse")
+ }
+
+ val numFeatures = dataset.select(col($(featuresCol))).limit(1).map {
+ case Row(features: Vector) => features.size
+ }.first()
+ if (numFeatures > 4096) {
+ val msg = "Currently, GeneralizedLinearRegression only supports number of features" +
+ s" <= 4096. Found $numFeatures in the input dataset."
+ throw new SparkException(msg)
+ }
+
+ val w = if ($(weightCol).isEmpty) lit(1.0) else col($(weightCol))
+ val instances: RDD[Instance] = dataset.select(
+ col($(labelCol)), w, col($(featuresCol))).map {
+ case Row(label: Double, weight: Double, features: Vector) =>
+ Instance(label, weight, features)
+ }
+
+ if ($(family) == "gaussian" && $(link) == "identity") {
+ val optimizer = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true)
+ val wlsModel = optimizer.fit(instances)
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ wlsModel.coefficients, wlsModel.intercept).setParent(this))
+ return model
+ }
+
+ val newInstances = instances.map { instance =>
+ val mu = familyLink.initialize(instance.label, instance.weight)
+ val eta = familyLink.predict(mu)
+ Instance(eta, instance.weight, instance.features)
+ }
+
+ val initialModel = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true).fit(newInstances)
+
+ val reweightFunc: (Instance, WeightedLeastSquaresModel) => (Double, Double) = {
+ (instance: Instance, model: WeightedLeastSquaresModel) => {
+ val eta = model.predict(instance.features)
+ val mu = familyLink.fitted(eta)
+ val z = familyLink.adjusted(instance.label, mu, eta)
+ val w = familyLink.weights(mu) * instance.weight
+ (z, w)
+ }
+ }
+
+ val optimizer = new IterativelyReweightedLeastSquares(initialModel, reweightFunc,
+ $(fitIntercept), $(regParam), $(maxIter), $(tol))
+
+ val irlsModel = optimizer.fit(instances)
+
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ irlsModel.coefficients, irlsModel.intercept).setParent(this))
+ model
+ }
+
+ @Since("2.0.0")
+ override def copy(extra: ParamMap): GeneralizedLinearRegression = defaultCopy(extra)
+}
+
+@Since("2.0.0")
+object GeneralizedLinearRegression {
+
+ /** Set of families that GeneralizedLinearRegression supports */
+ private[ml] val supportedFamilies = Set("gaussian", "binomial", "poisson", "gamma")
+
+ /** Set of links that GeneralizedLinearRegression supports */
+ private[ml] val supportedLinks = Set("identity", "log", "inverse", "logit", "probit",
+ "cloglog", "sqrt")
+
+ /** Set of family and link pairs that GeneralizedLinearRegression supports */
+ private[ml] val supportedFamilyLinkPairs = Set(
+ "gaussian" -> "identity", "gaussian" -> "log", "gaussian" -> "inverse",
+ "binomial" -> "logit", "binomial" -> "probit", "binomial" -> "cloglog",
+ "poisson" -> "log", "poisson" -> "identity", "poisson" -> "sqrt",
+ "gamma" -> "inverse", "gamma" -> "identity", "gamma" -> "log"
+ )
+}
+
+/**
+ * :: Experimental ::
+ * Model produced by [[GeneralizedLinearRegression]].
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegressionModel private[ml] (
+ @Since("2.0.0") override val uid: String,
+ @Since("2.0.0") val coefficients: Vector,
+ @Since("2.0.0") val intercept: Double)
+ extends RegressionModel[Vector, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams {
+
+ private lazy val familyLink = $(family) match {
+ case "gaussian" => if (isDefined(link)) Gaussian($(link)) else Gaussian("identity")
+ case "binomial" => if (isDefined(link)) Binomial($(link)) else Binomial("logit")
+ case "poisson" => if (isDefined(link)) Poisson($(link)) else Poisson("log")
+ case "gamma" => if (isDefined(link)) Gamma($(link)) else Gamma("inverse")
+ }
+
+ override protected def predict(features: Vector): Double = {
+ val eta = BLAS.dot(features, coefficients) + intercept
+ familyLink.fitted(eta)
+ }
+
+ @Since("2.0.0")
+ override def copy(extra: ParamMap): GeneralizedLinearRegressionModel = {
+ copyValues(new GeneralizedLinearRegressionModel(uid, coefficients, intercept), extra)
+ .setParent(parent)
+ }
+}
+
+/**
+ * A description of the error distribution and link function to be used in the model.
+ * @param link a link function instance
+ */
+private[ml] abstract class Family(val link: Link) extends Serializable {
+
+ /** Initialize the starting value for mu. */
+ def initialize(y: Double, weight: Double): Double
+
+ /** The variance of the endogenous variable's mean, given the value mu. */
+ def variance(mu: Double): Double
+
+ /** Weights for IRLS steps. */
+ def weights(mu: Double): Double = {
+ val x = clean(mu)
+ 1.0 / (math.pow(this.link.deriv(x), 2.0) * this.variance(x))
+ }
+
+ /** The adjusted response variable. */
+ def adjusted(y: Double, mu: Double, eta: Double): Double = {
+ val x = clean(mu)
+ eta + (y - x) * link.deriv(x)
+ }
+
+ /** Linear predictor based on given mu. */
+ def predict(mu: Double): Double = this.link.link(clean(mu))
+
+ /** Fitted value based on linear predictor eta. */
+ def fitted(eta: Double): Double = clean(this.link.unlink(eta))
+
+ /** Trim the fitted value so that it will be in valid range. */
+ def clean(mu: Double): Double = mu
+
+ val epsilon: Double = 1E-16
--- End diff --
If this is a constant, we should move it out.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-184162822
**[Test build #51306 has started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/51306/consoleFull)** for PR 11136 at commit [`4a27970`](https://github.com/apache/spark/commit/4a27970486091b6359b9d73ba477044d04c20875).
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-190538867
**[Test build #52216 has finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/52216/consoleFull)** for PR 11136 at commit [`31a912c`](https://github.com/apache/spark/commit/31a912cd74cf3dffbf8cc0af8c57b777d49579eb).
* This patch **fails to build**.
* This patch merges cleanly.
* This patch adds no public classes.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by AmplabJenkins <gi...@git.apache.org>.
Github user AmplabJenkins commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-188745905
Test PASSed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/51963/
Test PASSed.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-181927218
**[Test build #50978 has started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/50978/consoleFull)** for PR 11136 at commit [`5af604e`](https://github.com/apache/spark/commit/5af604e180d92a75b18b30717d6516d5ee8135bd).
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by yanboliang <gi...@git.apache.org>.
Github user yanboliang commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r53911596
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,565 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionBase extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * Default is "gaussian".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "The name of family which is a description of the error distribution to be used in the " +
+ "model. Supported options: gaussian(default), binomial, poisson and gamma.",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilyNames.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of link function which provides the relationship
+ * between the linear predictor and the mean of the distribution function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "The name of link function " +
+ "which provides the relationship between the linear predictor and the mean of the " +
+ "distribution function. Supported options: identity, log, inverse, logit, probit, " +
+ "cloglog and sqrt.",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinkNames.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ import GeneralizedLinearRegression._
+ protected lazy val familyObj = Family.fromName($(family))
+ protected lazy val linkObj = if (isDefined(link)) {
+ Link.fromName($(link))
+ } else {
+ familyObj.defaultLink
+ }
+ protected lazy val familyAndLink = new FamilyAndLink(familyObj, linkObj)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ if ($(solver) == "irls") {
+ setDefault(maxIter -> 25)
+ }
+ if (isDefined(link)) {
+ require(GeneralizedLinearRegression.supportedFamilyAndLinkParis.contains(
+ familyObj -> linkObj), s"Generalized Linear Regression with ${$(family)} family " +
+ s"does not support ${$(link)} link function.")
+ }
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor (link function) and
+ * a description of the error distribution (family).
+ * It supports "gaussian", "binomial", "poisson" and "gamma" as family.
+ * Valid link functions for each family is listed below. The first link function of each family
+ * is the default one.
+ * - "gaussian" -> "identity", "log", "inverse"
+ * - "binomial" -> "logit", "probit", "cloglog"
+ * - "poisson" -> "log", "identity", "sqrt"
+ * - "gamma" -> "inverse", "identity", "log"
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionBase with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("glm"))
+
+ /**
+ * Sets the value of param [[family]].
+ * Default is "gaussian".
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+ setDefault(family -> "gaussian")
+
+ /**
+ * Sets the value of param [[link]].
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Sets if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+
+ /**
+ * Sets the maximum number of iterations.
+ * Default is 25 if the solver algorithm is "irls".
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+
+ /**
+ * Sets the convergence tolerance of iterations.
+ * Smaller value will lead to higher accuracy with the cost of more iterations.
+ * Default is 1E-6.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setTol(value: Double): this.type = set(tol, value)
+ setDefault(tol -> 1E-6)
+
+ /**
+ * Sets the regularization parameter.
+ * Default is 0.0.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setRegParam(value: Double): this.type = set(regParam, value)
+ setDefault(regParam -> 0.0)
+
+ /**
+ * Sets the value of param [[weightCol]].
+ * If this is not set or empty, we treat all instance weights as 1.0.
+ * Default is empty, so all instances have weight one.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setWeightCol(value: String): this.type = set(weightCol, value)
+ setDefault(weightCol -> "")
+
+ /**
+ * Sets the solver algorithm used for optimization.
+ * Currently only support "irls" which is also the default solver.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setSolver(value: String): this.type = set(solver, value)
+ setDefault(solver -> "irls")
+
+ override protected def train(dataset: DataFrame): GeneralizedLinearRegressionModel = {
+ val numFeatures = dataset.select(col($(featuresCol))).limit(1)
+ .map { case Row(features: Vector) =>
+ features.size
+ }.first()
+ if (numFeatures > WeightedLeastSquares.MaxNumFeatures) {
+ val msg = "Currently, GeneralizedLinearRegression only supports number of features" +
+ s" <= ${WeightedLeastSquares.MaxNumFeatures}. Found $numFeatures in the input dataset."
+ throw new SparkException(msg)
+ }
+
+ val w = if ($(weightCol).isEmpty) lit(1.0) else col($(weightCol))
+ val instances: RDD[Instance] = dataset.select(col($(labelCol)), w, col($(featuresCol)))
+ .map { case Row(label: Double, weight: Double, features: Vector) =>
+ Instance(label, weight, features)
+ }
+
+ if (familyObj == GeneralizedLinearRegression.Gaussian &&
+ linkObj == GeneralizedLinearRegression.Identity) {
+ // TODO: Make standardizeFeatures and standardizeLabel configurable.
+ val optimizer = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true)
+ val wlsModel = optimizer.fit(instances)
+ val model = copyValues(
+ new GeneralizedLinearRegressionModel(uid, wlsModel.coefficients, wlsModel.intercept)
+ .setParent(this))
+ return model
+ }
+
+ // Fit Generalized Linear Model by iteratively reweighted least squares (IRLS).
+ val initialModel = familyAndLink.initialize(instances, $(fitIntercept), $(regParam))
+ val optimizer = new IterativelyReweightedLeastSquares(initialModel, familyAndLink.reweightFunc,
+ $(fitIntercept), $(regParam), $(maxIter), $(tol))
+ val irlsModel = optimizer.fit(instances)
+
+ val model = copyValues(
+ new GeneralizedLinearRegressionModel(uid, irlsModel.coefficients, irlsModel.intercept)
+ .setParent(this))
+ model
+ }
+
+ @Since("2.0.0")
+ override def copy(extra: ParamMap): GeneralizedLinearRegression = defaultCopy(extra)
+}
+
+@Since("2.0.0")
+private[ml] object GeneralizedLinearRegression {
+
+ /** Set of family and link pairs that GeneralizedLinearRegression supports. */
+ lazy val supportedFamilyAndLinkParis = Set(
+ Gaussian -> Identity, Gaussian -> Log, Gaussian -> Inverse,
+ Binomial -> Logit, Binomial -> Probit, Binomial -> CLogLog,
+ Poisson -> Log, Poisson -> Identity, Poisson -> Sqrt,
+ Gamma -> Inverse, Gamma -> Identity, Gamma -> Log
+ )
+
+ /** Set of family names that GeneralizedLinearRegression supports. */
+ lazy val supportedFamilyNames = supportedFamilyAndLinkParis.map(_._1.name)
+
+ /** Set of link names that GeneralizedLinearRegression supports. */
+ lazy val supportedLinkNames = supportedFamilyAndLinkParis.map(_._2.name)
+
+ val epsilon: Double = 1E-16
+
+ /**
+ * Wrapper of family and link combination used in the model.
+ */
+ private[ml] class FamilyAndLink(val family: Family, var link: Link) extends Serializable {
+
+ /** Linear predictor based on given mu. */
+ def predict(mu: Double): Double = link.link(family.project(mu))
+
+ /** Fitted value based on linear predictor eta. */
+ def fitted(eta: Double): Double = family.project(link.unlink(eta))
+
+ /**
+ * Get the initial guess model for [[IterativelyReweightedLeastSquares]].
+ */
+ def initialize(
+ instances: RDD[Instance],
+ fitIntercept: Boolean,
+ regParam: Double): WeightedLeastSquaresModel = {
+ val newInstances = instances.map { instance =>
+ val mu = family.initialize(instance.label, instance.weight)
+ val eta = predict(mu)
+ Instance(eta, instance.weight, instance.features)
+ }
+ // TODO: Make standardizeFeatures and standardizeLabel configurable.
+ val initialModel = new WeightedLeastSquares(fitIntercept, regParam,
+ standardizeFeatures = true, standardizeLabel = true)
+ .fit(newInstances)
+ initialModel
+ }
+
+ /**
+ * The reweight function used to update offsets and weights
+ * at each iteration of [[IterativelyReweightedLeastSquares]].
+ */
+ val reweightFunc: (Instance, WeightedLeastSquaresModel) => (Double, Double) = {
+ (instance: Instance, model: WeightedLeastSquaresModel) => {
+ val eta = model.predict(instance.features)
+ val mu = fitted(eta)
+ val offset = eta + (instance.label - mu) * link.deriv(mu)
+ val weight = instance.weight / (math.pow(this.link.deriv(mu), 2.0) * family.variance(mu))
+ (offset, weight)
+ }
+ }
+ }
+
+ /**
+ * A description of the error distribution to be used in the model.
+ * @param name the name of the family.
+ */
+ private[ml] abstract class Family(val name: String) extends Serializable {
+
+ /** The default link instance of this family. */
+ val defaultLink: Link
+
+ /** Initialize the starting value for mu. */
+ def initialize(y: Double, weight: Double): Double
+
+ /** The variance of the endogenous variable's mean, given the value mu. */
+ def variance(mu: Double): Double
+
+ /** Trim the fitted value so that it will be in valid range. */
+ def project(mu: Double): Double = mu
--- End diff --
Here we constrict ```mu``` in valid range using a different method compared with R. R use ```validmu``` and ```valideta``` to check whether ```mu``` and ```eta``` in valid range. If ```mu``` or ```eta``` is invalid, it will diminish coefficients until it makes ```validmu``` and ```validaeta``` passed. I think is will make convergence slowness.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by shaneknapp <gi...@git.apache.org>.
Github user shaneknapp commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-182936359
jenkins, test this please
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by sethah <gi...@git.apache.org>.
Github user sethah commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r52401115
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,472 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.Logging
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
+ s"Generalized Linear Regression with ${$(family)} family does not support ${$(link)} " +
+ s"link function.")
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor and
+ * a description of the error distribution.
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("genLinReg"))
+
+ /**
+ * Set the name of family which is a description of the error distribution
+ * to be used in the model.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+
+ /**
+ * Set the name of the model link function.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Set if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+ setDefault(fitIntercept -> true)
+
+ /**
+ * Set the maximum number of iterations.
+ * Default is 100.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+ setDefault(maxIter -> 100)
+
+ /**
+ * Set the convergence tolerance of iterations.
+ * Smaller value will lead to higher accuracy with the cost of more iterations.
+ * Default is 1E-6.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setTol(value: Double): this.type = set(tol, value)
+ setDefault(tol -> 1E-6)
+
+ /**
+ * Set the regularization parameter.
+ * Default is 0.0.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setRegParam(value: Double): this.type = set(regParam, value)
+ setDefault(regParam -> 0.0)
+
+ /**
+ * Whether to over-/under-sample training instances according to the given weights in weightCol.
+ * If empty, all instances are treated equally (weight 1.0).
+ * Default is empty, so all instances have weight one.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setWeightCol(value: String): this.type = set(weightCol, value)
+ setDefault(weightCol -> "")
+
+ /**
+ * Set the solver algorithm used for optimization.
+ * Currently only support "irls" which is also the default solver.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setSolver(value: String): this.type = set(solver, value)
+ setDefault(solver -> "irls")
+
+ override protected def train(dataset: DataFrame): GeneralizedLinearRegressionModel = {
+ val familyLink = $(family) match {
+ case "gaussian" => Gaussian($(link))
+ case "binomial" => Binomial($(link))
+ case "poisson" => Poisson($(link))
+ case "gamma" => Gamma($(link))
+ }
+
+ val w = if ($(weightCol).isEmpty) lit(1.0) else col($(weightCol))
+ val instances: RDD[Instance] = dataset.select(
+ col($(labelCol)), w, col($(featuresCol))).map {
+ case Row(label: Double, weight: Double, features: Vector) =>
+ Instance(label, weight, features)
+ }
+
+ if ($(family) == "gaussian" && $(link) == "identity") {
+ val optimizer = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true)
+ val wlsModel = optimizer.fit(instances)
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ wlsModel.coefficients, wlsModel.intercept).setParent(this))
+ return model
+ }
+
+ val newInstances = instances.map { instance =>
+ val mu = familyLink.initialize(instance.label, instance.weight)
+ val eta = familyLink.predict(mu)
+ Instance(eta, instance.weight, instance.features)
+ }
+
+ val initialModel = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true).fit(newInstances)
+
+ val reweightFunc: (Instance, WeightedLeastSquaresModel) => (Double, Double) = {
+ (instance: Instance, model: WeightedLeastSquaresModel) => {
+ val eta = model.predict(instance.features)
+ val mu = familyLink.fitted(eta)
+ val z = familyLink.adjusted(instance.label, mu, eta)
+ val w = familyLink.weights(mu) * instance.weight
+ (z, w)
+ }
+ }
+
+ val optimizer = new IterativelyReweightedLeastSquares(initialModel, reweightFunc,
+ $(fitIntercept), $(regParam), $(maxIter), $(tol))
+
+ val irlsModel = optimizer.fit(instances)
+
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ irlsModel.coefficients, irlsModel.intercept).setParent(this))
+ model
+ }
+
+ @Since("2.0.0")
+ override def copy(extra: ParamMap): GeneralizedLinearRegression = defaultCopy(extra)
+}
+
+@Since("2.0.0")
+object GeneralizedLinearRegression {
+
+ /** Set of families that GeneralizedLinearRegression supports */
+ private[ml] val supportedFamilies = Set("gaussian", "binomial", "poisson", "gamma")
+
+ /** Set of links that GeneralizedLinearRegression supports */
+ private[ml] val supportedLinks = Set("identity", "log", "inverse", "logit", "probit",
+ "cloglog", "sqrt")
+
+ /** Set of family and link pairs that GeneralizedLinearRegression supports */
+ private[ml] val supportedFamilyLinkPairs = Set(
+ "gaussian" -> "identity", "gaussian" -> "log", "gaussian" -> "inverse",
+ "binomial" -> "logit", "binomial" -> "probit", "binomial" -> "cloglog",
+ "poisson" -> "log", "poisson" -> "identity", "poisson" -> "sqrt",
+ "gamma" -> "inverse", "gamma" -> "identity", "gamma" -> "log"
+ )
+}
+
+/**
+ * :: Experimental ::
+ * Model produced by [[GeneralizedLinearRegression]].
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegressionModel private[ml] (
+ @Since("2.0.0") override val uid: String,
+ @Since("2.0.0") val coefficients: Vector,
+ @Since("2.0.0") val intercept: Double)
+ extends RegressionModel[Vector, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams {
+
+ private lazy val familyLink = $(family) match {
+ case "gaussian" => Gaussian($(link))
+ case "binomial" => Binomial($(link))
+ case "poisson" => Poisson($(link))
+ case "gamma" => Gamma($(link))
+ }
+
+ override protected def predict(features: Vector): Double = {
+ val eta = BLAS.dot(features, coefficients) + intercept
+ familyLink.fitted(eta)
+ }
+
+ @Since("2.0.0")
+ override def copy(extra: ParamMap): GeneralizedLinearRegressionModel = {
+ copyValues(new GeneralizedLinearRegressionModel(uid, coefficients, intercept), extra)
+ .setParent(parent)
+ }
+}
+
+/**
+ * A description of the error distribution and link function to be used in the model.
+ * @param link a link function instance
+ */
+private[ml] abstract class Family(val link: Link) extends Serializable {
+
+ /** Initialize the starting value for mu. */
+ def initialize(y: Double, weight: Double): Double
+
+ /** The variance of mu to its mean. */
--- End diff --
I think this comment is a bit confusing. This is really the variance of the endogenous variable, given the expected value `mu`.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by yanboliang <gi...@git.apache.org>.
Github user yanboliang commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r53911268
--- Diff: mllib/src/test/scala/org/apache/spark/ml/regression/GeneralizedLinearRegressionSuite.scala ---
@@ -0,0 +1,499 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import scala.util.Random
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.ml.param.ParamsSuite
+import org.apache.spark.ml.util.MLTestingUtils
+import org.apache.spark.mllib.classification.LogisticRegressionSuite._
+import org.apache.spark.mllib.linalg.{BLAS, DenseVector, Vectors}
+import org.apache.spark.mllib.random._
+import org.apache.spark.mllib.regression.LabeledPoint
+import org.apache.spark.mllib.util.MLlibTestSparkContext
+import org.apache.spark.mllib.util.TestingUtils._
+import org.apache.spark.sql.{DataFrame, Row}
+
+class GeneralizedLinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
+
+ private val seed: Int = 42
+ @transient var datasetGaussianIdentity: DataFrame = _
+ @transient var datasetGaussianLog: DataFrame = _
+ @transient var datasetGaussianInverse: DataFrame = _
+ @transient var datasetBinomial: DataFrame = _
+ @transient var datasetPoissonLog: DataFrame = _
+ @transient var datasetPoissonIdentity: DataFrame = _
+ @transient var datasetPoissonSqrt: DataFrame = _
+ @transient var datasetGammaInverse: DataFrame = _
+ @transient var datasetGammaIdentity: DataFrame = _
+ @transient var datasetGammaLog: DataFrame = _
+
+ override def beforeAll(): Unit = {
+ super.beforeAll()
+
+ import GeneralizedLinearRegressionSuite._
+
+ datasetGaussianIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "identity"), 2))
+
+ datasetGaussianLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "log"), 2))
+
+ datasetGaussianInverse = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "inverse"), 2))
+
+ datasetBinomial = {
+ val nPoints = 10000
+ val coefficients = Array(-0.57997, 0.912083, -0.371077, -0.819866, 2.688191)
+ val xMean = Array(5.843, 3.057, 3.758, 1.199)
+ val xVariance = Array(0.6856, 0.1899, 3.116, 0.581)
+
+ val testData =
+ generateMultinomialLogisticInput(coefficients, xMean, xVariance, true, nPoints, seed)
+
+ sqlContext.createDataFrame(sc.parallelize(testData, 4))
+ }
+
+ datasetPoissonLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "log"), 2))
+
+ datasetPoissonIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "identity"), 2))
+
+ datasetPoissonSqrt = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "sqrt"), 2))
+
+ datasetGammaInverse = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "inverse"), 2))
+
+ datasetGammaIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "identity"), 2))
+
+ datasetGammaLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "log"), 2))
+ }
+
+ test("params") {
+ ParamsSuite.checkParams(new GeneralizedLinearRegression)
+ val model = new GeneralizedLinearRegressionModel("genLinReg", Vectors.dense(0.0), 0.0)
+ ParamsSuite.checkParams(model)
+ }
+
+ test("generalized linear regression: default params") {
+ val glr = new GeneralizedLinearRegression
+ assert(glr.getLabelCol === "label")
+ assert(glr.getFeaturesCol === "features")
+ assert(glr.getPredictionCol === "prediction")
+ assert(glr.getFitIntercept)
+ assert(glr.getTol === 1E-6)
+ assert(glr.getWeightCol === "")
+ assert(glr.getRegParam === 0.0)
+ assert(glr.getSolver == "irls")
+ val model = glr.setFamily("gaussian").setLink("identity")
+ .fit(datasetGaussianIdentity)
+
+ // copied model must have the same parent.
+ MLTestingUtils.checkCopy(model)
+ model.transform(datasetGaussianIdentity)
+ .select("label", "prediction")
+ .collect()
+
+ assert(model.getFeaturesCol === "features")
+ assert(model.getPredictionCol === "prediction")
+ assert(model.intercept !== 0.0)
+ assert(model.hasParent)
+ assert(model.getFamily === "gaussian")
+ assert(model.getLink === "identity")
+ }
+
+ test("generalized linear regression: gaussian family against glm") {
+ /*
+ R code:
+ f1 <- data$V1 ~ data$V2 + data$V3 - 1
+ f2 <- data$V1 ~ data$V2 + data$V3
+
+ data <- read.csv("path", header=FALSE)
+ for (formula in c(f1, f2)) {
+ model <- glm(formula, family="gaussian", data=data)
+ print(as.vector(coef(model)))
+ }
+
+ [1] 2.2960999 0.8087933
+ [1] 2.5002642 2.2000403 0.5999485
+
+ data <- read.csv("path", header=FALSE)
+ model1 <- glm(f1, family=gaussian(link=log), data=data, start=c(0,0))
+ model2 <- glm(f2, family=gaussian(link=log), data=data, start=c(0,0,0))
+ print(as.vector(coef(model1)))
+ print(as.vector(coef(model2)))
+
+ [1] 0.23069326 0.07993778
+ [1] 0.25001858 0.22002452 0.05998789
+
+ data <- read.csv("path", header=FALSE)
+ for (formula in c(f1, f2)) {
+ model <- glm(formula, family=gaussian(link=inverse), data=data)
+ print(as.vector(coef(model)))
+ }
+
+ [1] 2.3010179 0.8198976
+ [1] 2.4108902 2.2130248 0.6086152
+ */
+
+ val expected = Seq(
+ Vectors.dense(0.0, 2.2960999, 0.8087933),
+ Vectors.dense(2.5002642, 2.2000403, 0.5999485),
+ Vectors.dense(0.0, 0.23069326, 0.07993778),
+ Vectors.dense(0.25001858, 0.22002452, 0.05998789),
+ Vectors.dense(0.0, 2.3010179, 0.8198976),
+ Vectors.dense(2.4108902, 2.2130248, 0.6086152))
+
+ import GeneralizedLinearRegression._
+
+ var idx = 0
+ for ((link, dataset) <- Seq(("identity", datasetGaussianIdentity), ("log", datasetGaussianLog),
+ ("inverse", datasetGaussianInverse))) {
+ for (fitIntercept <- Seq(false, true)) {
+ val trainer = new GeneralizedLinearRegression().setFamily("gaussian").setLink(link)
+ .setFitIntercept(fitIntercept)
+ val model = trainer.fit(dataset)
+ val actual = Vectors.dense(model.intercept, model.coefficients(0), model.coefficients(1))
+ assert(actual ~== expected(idx) absTol 1e-4)
+
+ val familyLink = new FamilyAndLink(Gaussian, Link.fromName(link))
+ model.transform(dataset).select("features", "prediction").collect().foreach {
+ case Row(features: DenseVector, prediction1: Double) =>
+ val eta = BLAS.dot(features, model.coefficients) + model.intercept
+ val prediction2 = familyLink.fitted(eta)
+ assert(prediction1 ~== prediction2 relTol 1E-5)
+ }
+
+ idx += 1
+ }
+ }
+ }
+
+ test("generalized linear regression: gaussian family against glmnet") {
--- End diff --
I only added test for ```gaussian family against glmnet``` to check the result of enable regularization. For other families, the models are not exactly match under restrictions of ```1e-4```. R glm and glmnet use different optimization method, and I think it needs be discussed and solved by follow up work.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r53722430
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,547 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ if (isDefined(link)) {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
+ s"Generalized Linear Regression with ${$(family)} family does not support ${$(link)} " +
+ s"link function.")
+ }
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor and
+ * a description of the error distribution.
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("genLinReg"))
+
+ /**
+ * Set the name of family which is a description of the error distribution
+ * to be used in the model.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+
+ /**
+ * Set the name of the model link function.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Set if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+ setDefault(fitIntercept -> true)
--- End diff --
This is not necessary. It is already set to true by default in `HasFitIntercept`.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by sethah <gi...@git.apache.org>.
Github user sethah commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r52539912
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,472 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.Logging
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
+ s"Generalized Linear Regression with ${$(family)} family does not support ${$(link)} " +
+ s"link function.")
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor and
+ * a description of the error distribution.
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("genLinReg"))
+
+ /**
+ * Set the name of family which is a description of the error distribution
+ * to be used in the model.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+
+ /**
+ * Set the name of the model link function.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Set if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+ setDefault(fitIntercept -> true)
+
+ /**
+ * Set the maximum number of iterations.
+ * Default is 100.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+ setDefault(maxIter -> 100)
+
+ /**
+ * Set the convergence tolerance of iterations.
+ * Smaller value will lead to higher accuracy with the cost of more iterations.
+ * Default is 1E-6.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setTol(value: Double): this.type = set(tol, value)
+ setDefault(tol -> 1E-6)
+
+ /**
+ * Set the regularization parameter.
+ * Default is 0.0.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setRegParam(value: Double): this.type = set(regParam, value)
+ setDefault(regParam -> 0.0)
+
+ /**
+ * Whether to over-/under-sample training instances according to the given weights in weightCol.
+ * If empty, all instances are treated equally (weight 1.0).
+ * Default is empty, so all instances have weight one.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setWeightCol(value: String): this.type = set(weightCol, value)
+ setDefault(weightCol -> "")
+
+ /**
+ * Set the solver algorithm used for optimization.
+ * Currently only support "irls" which is also the default solver.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setSolver(value: String): this.type = set(solver, value)
+ setDefault(solver -> "irls")
+
+ override protected def train(dataset: DataFrame): GeneralizedLinearRegressionModel = {
+ val familyLink = $(family) match {
+ case "gaussian" => Gaussian($(link))
+ case "binomial" => Binomial($(link))
+ case "poisson" => Poisson($(link))
+ case "gamma" => Gamma($(link))
+ }
+
+ val w = if ($(weightCol).isEmpty) lit(1.0) else col($(weightCol))
+ val instances: RDD[Instance] = dataset.select(
+ col($(labelCol)), w, col($(featuresCol))).map {
+ case Row(label: Double, weight: Double, features: Vector) =>
+ Instance(label, weight, features)
+ }
+
+ if ($(family) == "gaussian" && $(link) == "identity") {
+ val optimizer = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true)
+ val wlsModel = optimizer.fit(instances)
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ wlsModel.coefficients, wlsModel.intercept).setParent(this))
+ return model
+ }
+
+ val newInstances = instances.map { instance =>
+ val mu = familyLink.initialize(instance.label, instance.weight)
+ val eta = familyLink.predict(mu)
+ Instance(eta, instance.weight, instance.features)
+ }
+
+ val initialModel = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true).fit(newInstances)
+
+ val reweightFunc: (Instance, WeightedLeastSquaresModel) => (Double, Double) = {
+ (instance: Instance, model: WeightedLeastSquaresModel) => {
+ val eta = model.predict(instance.features)
+ val mu = familyLink.fitted(eta)
+ val z = familyLink.adjusted(instance.label, mu, eta)
+ val w = familyLink.weights(mu) * instance.weight
+ (z, w)
+ }
+ }
+
+ val optimizer = new IterativelyReweightedLeastSquares(initialModel, reweightFunc,
+ $(fitIntercept), $(regParam), $(maxIter), $(tol))
+
+ val irlsModel = optimizer.fit(instances)
+
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ irlsModel.coefficients, irlsModel.intercept).setParent(this))
+ model
+ }
+
+ @Since("2.0.0")
+ override def copy(extra: ParamMap): GeneralizedLinearRegression = defaultCopy(extra)
+}
+
+@Since("2.0.0")
+object GeneralizedLinearRegression {
+
+ /** Set of families that GeneralizedLinearRegression supports */
+ private[ml] val supportedFamilies = Set("gaussian", "binomial", "poisson", "gamma")
+
+ /** Set of links that GeneralizedLinearRegression supports */
+ private[ml] val supportedLinks = Set("identity", "log", "inverse", "logit", "probit",
+ "cloglog", "sqrt")
+
+ /** Set of family and link pairs that GeneralizedLinearRegression supports */
+ private[ml] val supportedFamilyLinkPairs = Set(
+ "gaussian" -> "identity", "gaussian" -> "log", "gaussian" -> "inverse",
+ "binomial" -> "logit", "binomial" -> "probit", "binomial" -> "cloglog",
+ "poisson" -> "log", "poisson" -> "identity", "poisson" -> "sqrt",
+ "gamma" -> "inverse", "gamma" -> "identity", "gamma" -> "log"
+ )
+}
+
+/**
+ * :: Experimental ::
+ * Model produced by [[GeneralizedLinearRegression]].
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegressionModel private[ml] (
+ @Since("2.0.0") override val uid: String,
+ @Since("2.0.0") val coefficients: Vector,
+ @Since("2.0.0") val intercept: Double)
+ extends RegressionModel[Vector, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams {
+
+ private lazy val familyLink = $(family) match {
+ case "gaussian" => Gaussian($(link))
+ case "binomial" => Binomial($(link))
+ case "poisson" => Poisson($(link))
+ case "gamma" => Gamma($(link))
+ }
+
+ override protected def predict(features: Vector): Double = {
+ val eta = BLAS.dot(features, coefficients) + intercept
+ familyLink.fitted(eta)
+ }
+
+ @Since("2.0.0")
+ override def copy(extra: ParamMap): GeneralizedLinearRegressionModel = {
+ copyValues(new GeneralizedLinearRegressionModel(uid, coefficients, intercept), extra)
+ .setParent(parent)
+ }
+}
+
+/**
+ * A description of the error distribution and link function to be used in the model.
+ * @param link a link function instance
+ */
+private[ml] abstract class Family(val link: Link) extends Serializable {
+
+ /** Initialize the starting value for mu. */
+ def initialize(y: Double, weight: Double): Double
+
+ /** The variance of mu to its mean. */
+ def variance(mu: Double): Double
+
+ /** Weights for IRLS steps. */
+ def weights(mu: Double): Double = {
+ 1.0 / (math.pow(this.link.deriv(mu), 2.0) * this.variance(mu))
+ }
+
+ /** The adjusted response variable. */
+ def adjusted(y: Double, mu: Double, eta: Double): Double = {
+ eta + (y - mu) * link.deriv(mu)
+ }
+
+ /** Linear predictors based on given mu. */
+ def predict(mu: Double): Double = this.link.link(mu)
+
+ /** Fitted values based on linear predictors eta. */
+ def fitted(eta: Double): Double = this.link.unlink(eta)
+}
+
+/**
+ * Gaussian exponential family distribution.
+ * The default link for the Gaussian family is the identity link.
+ * @param link a link function instance
+ */
+private[ml] class Gaussian(link: Link = Identity) extends Family(link) {
+
+ override def initialize(y: Double, weight: Double): Double = y
+
+ def variance(mu: Double): Double = 1.0
+}
+
+private[ml] object Gaussian {
+
+ def apply(link: String): Gaussian = {
+ link match {
+ case "identity" => new Gaussian(Identity)
+ case "log" => new Gaussian(Log)
+ case "inverse" => new Gaussian(Inverse)
+ }
+ }
+}
+
+/**
+ * Binomial exponential family distribution.
+ * The default link for the Binomial family is the logit link.
+ * @param link a link function instance
+ */
+private[ml] class Binomial(link: Link = Logit) extends Family(link) {
+
+ override def initialize(y: Double, weight: Double): Double = {
+ (weight * y + 0.5) / (weight + 1.0)
+ }
+
+ override def variance(mu: Double): Double = mu * (1 - mu)
+}
+
+private[ml] object Binomial {
+
+ def apply(link: String): Binomial = {
+ link match {
+ case "logit" => new Binomial(Logit)
+ case "probit" => new Binomial(Probit)
+ case "cloglog" => new Binomial(CLogLog)
+ }
+ }
+}
+
+/**
+ * Poisson exponential family distribution.
+ * The default link for the Poisson family is the log link.
+ * @param link a link function instance
+ */
+private[ml] class Poisson(link: Link = Log) extends Family(link) {
+
+ override def initialize(y: Double, weight: Double): Double = y + 0.1
+
+ override def variance(mu: Double): Double = mu
+}
+
+private[ml] object Poisson {
+
+ def apply(link: String): Poisson = {
+ link match {
+ case "log" => new Poisson(Log)
+ case "sqrt" => new Poisson(Sqrt)
+ case "identity" => new Poisson(Identity)
+ }
+ }
+}
+
+/**
+ * Gamma exponential family distribution.
+ * The default link for the Gamma family is the inverse link.
+ * @param link a link function instance
+ */
+private[ml] class Gamma(link: Link = Log) extends Family(link) {
+
+ override def initialize(y: Double, weight: Double): Double = y
+
+ override def variance(mu: Double): Double = math.pow(mu, 2.0)
+}
+
+private[ml] object Gamma {
+
+ def apply(link: String): Gamma = {
+ link match {
+ case "inverse" => new Gamma(Inverse)
+ case "identity" => new Gamma(Identity)
+ case "log" => new Gamma(Log)
+ }
+ }
+}
+
+/**
+ * A description of the link function to be used in the model.
+ */
+private[ml] trait Link extends Serializable {
+
+ /** The link function. */
+ def link(mu: Double): Double
+
+ /** The derivative function. */
+ def deriv(mu: Double): Double
+
+ /** The inverse link function. */
+ def unlink(eta: Double): Double
+}
+
+private[ml] object Identity extends Link {
+
+ override def link(mu: Double): Double = mu
+
+ override def deriv(mu: Double): Double = 1.0
+
+ override def unlink(eta: Double): Double = eta
+}
+
+private[ml] object Logit extends Link {
+
+ override def link(mu: Double): Double = math.log(mu / (1.0 - mu))
+
+ override def deriv(mu: Double): Double = 1.0 / (mu * (1.0 - mu))
+
+ override def unlink(eta: Double): Double = 1.0 / (1.0 + math.exp(-1.0 * eta))
+}
--- End diff --
I think the restriction on endogenous variable should go into the `Family` class since it is truly the distribution on Y that restricts the values. This is how R does it.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by AmplabJenkins <gi...@git.apache.org>.
Github user AmplabJenkins commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-182961038
Test PASSed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/51109/
Test PASSed.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r54205705
--- Diff: mllib/src/main/scala/org/apache/spark/ml/optim/WeightedLeastSquares.scala ---
@@ -157,6 +157,12 @@ private[ml] class WeightedLeastSquares(
private[ml] object WeightedLeastSquares {
/**
+ * In order to take the normal equation approach efficiently, [[WeightedLeastSquares]]
+ * only supports the number of features is no more than 4096.
+ */
+ val MaxNumFeatures: Int = 4096
--- End diff --
This is not specified in Spark Code Style guide and Scala code style guide recommends `MaxNumFeatures`. But I do like `MAX_NUM_FEATURES` better.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-182960725
**[Test build #51109 has finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/51109/consoleFull)** for PR 11136 at commit [`97c3f6a`](https://github.com/apache/spark/commit/97c3f6a2b94d6705606dd020f61c5c604b562b4c).
* This patch passes all tests.
* This patch merges cleanly.
* This patch adds no public classes.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by AmplabJenkins <gi...@git.apache.org>.
Github user AmplabJenkins commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-190522461
Merged build finished. Test FAILed.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-190607102
**[Test build #52227 has finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/52227/consoleFull)** for PR 11136 at commit [`007a4ec`](https://github.com/apache/spark/commit/007a4ec324db273c048ed65fe8942daba0c9d844).
* This patch passes all tests.
* This patch merges cleanly.
* This patch adds no public classes.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by AmplabJenkins <gi...@git.apache.org>.
Github user AmplabJenkins commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-190538871
Merged build finished. Test FAILed.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-190477077
I made one pass on the tests, only some minor comments.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r53722396
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,547 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ if (isDefined(link)) {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
+ s"Generalized Linear Regression with ${$(family)} family does not support ${$(link)} " +
+ s"link function.")
+ }
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor and
+ * a description of the error distribution.
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("genLinReg"))
--- End diff --
"glm", which is quite common
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r54508370
--- Diff: mllib/src/test/scala/org/apache/spark/ml/regression/GeneralizedLinearRegressionSuite.scala ---
@@ -0,0 +1,499 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import scala.util.Random
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.ml.param.ParamsSuite
+import org.apache.spark.ml.util.MLTestingUtils
+import org.apache.spark.mllib.classification.LogisticRegressionSuite._
+import org.apache.spark.mllib.linalg.{BLAS, DenseVector, Vectors}
+import org.apache.spark.mllib.random._
+import org.apache.spark.mllib.regression.LabeledPoint
+import org.apache.spark.mllib.util.MLlibTestSparkContext
+import org.apache.spark.mllib.util.TestingUtils._
+import org.apache.spark.sql.{DataFrame, Row}
+
+class GeneralizedLinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
+
+ private val seed: Int = 42
+ @transient var datasetGaussianIdentity: DataFrame = _
+ @transient var datasetGaussianLog: DataFrame = _
+ @transient var datasetGaussianInverse: DataFrame = _
+ @transient var datasetBinomial: DataFrame = _
+ @transient var datasetPoissonLog: DataFrame = _
+ @transient var datasetPoissonIdentity: DataFrame = _
+ @transient var datasetPoissonSqrt: DataFrame = _
+ @transient var datasetGammaInverse: DataFrame = _
+ @transient var datasetGammaIdentity: DataFrame = _
+ @transient var datasetGammaLog: DataFrame = _
+
+ override def beforeAll(): Unit = {
+ super.beforeAll()
+
+ import GeneralizedLinearRegressionSuite._
+
+ datasetGaussianIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "identity"), 2))
+
+ datasetGaussianLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "log"), 2))
+
+ datasetGaussianInverse = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "inverse"), 2))
+
+ datasetBinomial = {
+ val nPoints = 10000
+ val coefficients = Array(-0.57997, 0.912083, -0.371077, -0.819866, 2.688191)
+ val xMean = Array(5.843, 3.057, 3.758, 1.199)
+ val xVariance = Array(0.6856, 0.1899, 3.116, 0.581)
+
+ val testData =
+ generateMultinomialLogisticInput(coefficients, xMean, xVariance, true, nPoints, seed)
+
+ sqlContext.createDataFrame(sc.parallelize(testData, 4))
--- End diff --
Why using 4 partitions instead of 2, which is used in other datasets?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by yanboliang <gi...@git.apache.org>.
Github user yanboliang commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r53592447
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,547 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ if (isDefined(link)) {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
+ s"Generalized Linear Regression with ${$(family)} family does not support ${$(link)} " +
+ s"link function.")
+ }
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor and
+ * a description of the error distribution.
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("genLinReg"))
+
+ /**
+ * Set the name of family which is a description of the error distribution
+ * to be used in the model.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+
+ /**
+ * Set the name of the model link function.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Set if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+ setDefault(fitIntercept -> true)
+
+ /**
+ * Set the maximum number of iterations.
+ * Default is 100.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+ setDefault(maxIter -> 100)
+
+ /**
+ * Set the convergence tolerance of iterations.
+ * Smaller value will lead to higher accuracy with the cost of more iterations.
+ * Default is 1E-6.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setTol(value: Double): this.type = set(tol, value)
+ setDefault(tol -> 1E-6)
+
+ /**
+ * Set the regularization parameter.
+ * Default is 0.0.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setRegParam(value: Double): this.type = set(regParam, value)
+ setDefault(regParam -> 0.0)
+
+ /**
+ * Whether to over-/under-sample training instances according to the given weights in weightCol.
+ * If empty, all instances are treated equally (weight 1.0).
+ * Default is empty, so all instances have weight one.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setWeightCol(value: String): this.type = set(weightCol, value)
+ setDefault(weightCol -> "")
+
+ /**
+ * Set the solver algorithm used for optimization.
+ * Currently only support "irls" which is also the default solver.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setSolver(value: String): this.type = set(solver, value)
+ setDefault(solver -> "irls")
+
+ override protected def train(dataset: DataFrame): GeneralizedLinearRegressionModel = {
+ val familyLink = $(family) match {
+ case "gaussian" => if (isDefined(link)) Gaussian($(link)) else Gaussian("identity")
+ case "binomial" => if (isDefined(link)) Binomial($(link)) else Binomial("logit")
+ case "poisson" => if (isDefined(link)) Poisson($(link)) else Poisson("log")
+ case "gamma" => if (isDefined(link)) Gamma($(link)) else Gamma("inverse")
+ }
+
+ val numFeatures = dataset.select(col($(featuresCol))).limit(1).map {
+ case Row(features: Vector) => features.size
+ }.first()
+ if (numFeatures > 4096) {
+ val msg = "Currently, GeneralizedLinearRegression only supports number of features" +
+ s" <= 4096. Found $numFeatures in the input dataset."
+ throw new SparkException(msg)
+ }
+
+ val w = if ($(weightCol).isEmpty) lit(1.0) else col($(weightCol))
+ val instances: RDD[Instance] = dataset.select(
+ col($(labelCol)), w, col($(featuresCol))).map {
+ case Row(label: Double, weight: Double, features: Vector) =>
+ Instance(label, weight, features)
+ }
+
+ if ($(family) == "gaussian" && $(link) == "identity") {
+ val optimizer = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true)
+ val wlsModel = optimizer.fit(instances)
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ wlsModel.coefficients, wlsModel.intercept).setParent(this))
+ return model
+ }
+
+ val newInstances = instances.map { instance =>
+ val mu = familyLink.initialize(instance.label, instance.weight)
+ val eta = familyLink.predict(mu)
+ Instance(eta, instance.weight, instance.features)
+ }
+
+ val initialModel = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true).fit(newInstances)
+
+ val reweightFunc: (Instance, WeightedLeastSquaresModel) => (Double, Double) = {
+ (instance: Instance, model: WeightedLeastSquaresModel) => {
+ val eta = model.predict(instance.features)
+ val mu = familyLink.fitted(eta)
+ val z = familyLink.adjusted(instance.label, mu, eta)
+ val w = familyLink.weights(mu) * instance.weight
+ (z, w)
+ }
+ }
+
+ val optimizer = new IterativelyReweightedLeastSquares(initialModel, reweightFunc,
+ $(fitIntercept), $(regParam), $(maxIter), $(tol))
+
+ val irlsModel = optimizer.fit(instances)
+
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ irlsModel.coefficients, irlsModel.intercept).setParent(this))
+ model
+ }
+
+ @Since("2.0.0")
+ override def copy(extra: ParamMap): GeneralizedLinearRegression = defaultCopy(extra)
+}
+
+@Since("2.0.0")
+object GeneralizedLinearRegression {
+
+ /** Set of families that GeneralizedLinearRegression supports */
+ private[ml] val supportedFamilies = Set("gaussian", "binomial", "poisson", "gamma")
+
+ /** Set of links that GeneralizedLinearRegression supports */
+ private[ml] val supportedLinks = Set("identity", "log", "inverse", "logit", "probit",
+ "cloglog", "sqrt")
+
+ /** Set of family and link pairs that GeneralizedLinearRegression supports */
+ private[ml] val supportedFamilyLinkPairs = Set(
+ "gaussian" -> "identity", "gaussian" -> "log", "gaussian" -> "inverse",
+ "binomial" -> "logit", "binomial" -> "probit", "binomial" -> "cloglog",
+ "poisson" -> "log", "poisson" -> "identity", "poisson" -> "sqrt",
+ "gamma" -> "inverse", "gamma" -> "identity", "gamma" -> "log"
+ )
+}
+
+/**
+ * :: Experimental ::
+ * Model produced by [[GeneralizedLinearRegression]].
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegressionModel private[ml] (
+ @Since("2.0.0") override val uid: String,
+ @Since("2.0.0") val coefficients: Vector,
+ @Since("2.0.0") val intercept: Double)
+ extends RegressionModel[Vector, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams {
+
+ private lazy val familyLink = $(family) match {
+ case "gaussian" => if (isDefined(link)) Gaussian($(link)) else Gaussian("identity")
+ case "binomial" => if (isDefined(link)) Binomial($(link)) else Binomial("logit")
+ case "poisson" => if (isDefined(link)) Poisson($(link)) else Poisson("log")
+ case "gamma" => if (isDefined(link)) Gamma($(link)) else Gamma("inverse")
+ }
+
+ override protected def predict(features: Vector): Double = {
+ val eta = BLAS.dot(features, coefficients) + intercept
+ familyLink.fitted(eta)
+ }
+
+ @Since("2.0.0")
+ override def copy(extra: ParamMap): GeneralizedLinearRegressionModel = {
+ copyValues(new GeneralizedLinearRegressionModel(uid, coefficients, intercept), extra)
+ .setParent(parent)
+ }
+}
+
+/**
+ * A description of the error distribution and link function to be used in the model.
+ * @param link a link function instance
+ */
+private[ml] abstract class Family(val link: Link) extends Serializable {
+
+ /** Initialize the starting value for mu. */
+ def initialize(y: Double, weight: Double): Double
+
+ /** The variance of the endogenous variable's mean, given the value mu. */
+ def variance(mu: Double): Double
+
+ /** Weights for IRLS steps. */
+ def weights(mu: Double): Double = {
+ val x = clean(mu)
+ 1.0 / (math.pow(this.link.deriv(x), 2.0) * this.variance(x))
+ }
+
+ /** The adjusted response variable. */
+ def adjusted(y: Double, mu: Double, eta: Double): Double = {
+ val x = clean(mu)
+ eta + (y - x) * link.deriv(x)
+ }
+
+ /** Linear predictor based on given mu. */
+ def predict(mu: Double): Double = this.link.link(clean(mu))
+
+ /** Fitted value based on linear predictor eta. */
+ def fitted(eta: Double): Double = clean(this.link.unlink(eta))
+
+ /** Trim the fitted value so that it will be in valid range. */
+ def clean(mu: Double): Double = mu
+
+ val epsilon: Double = 1E-16
+}
+
+/**
+ * Gaussian exponential family distribution.
+ * The default link for the Gaussian family is the identity link.
+ * @param link a link function instance
+ */
+private[ml] class Gaussian(link: Link = Identity) extends Family(link) {
+
+ override def initialize(y: Double, weight: Double): Double = {
+ if (link == Log) {
+ require(y > 0.0, "The response variable of Gaussian family with Log link " +
+ s"should be positive, but got $y")
+ }
+ y
+ }
+
+ def variance(mu: Double): Double = 1.0
+
+ override def clean(mu: Double): Double = {
--- End diff --
Here we constrict ```mu``` in valid range using a different method compared with R. In R, if ```mu``` or ```eta``` is invalid, it will diminish ```coefficients``` until it makes ```validmu``` and ```validaeta``` passed. I think is will make convergence slowness. I'm looking forward to hear others' thought.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by AmplabJenkins <gi...@git.apache.org>.
Github user AmplabJenkins commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-184175089
Test PASSed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/51306/
Test PASSed.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by AmplabJenkins <gi...@git.apache.org>.
Github user AmplabJenkins commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-182897389
Merged build finished. Test FAILed.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r54059534
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,565 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionBase extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * Default is "gaussian".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "The name of family which is a description of the error distribution to be used in the " +
+ "model. Supported options: gaussian(default), binomial, poisson and gamma.",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilyNames.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of link function which provides the relationship
+ * between the linear predictor and the mean of the distribution function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "The name of link function " +
+ "which provides the relationship between the linear predictor and the mean of the " +
+ "distribution function. Supported options: identity, log, inverse, logit, probit, " +
+ "cloglog and sqrt.",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinkNames.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ import GeneralizedLinearRegression._
+ protected lazy val familyObj = Family.fromName($(family))
--- End diff --
This cannot be a member `val`. Users can set param `family` multiple times. We should move it and `linkObj` to `fit/train`.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by yanboliang <gi...@git.apache.org>.
Github user yanboliang commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r54518547
--- Diff: mllib/src/test/scala/org/apache/spark/ml/regression/GeneralizedLinearRegressionSuite.scala ---
@@ -0,0 +1,499 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import scala.util.Random
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.ml.param.ParamsSuite
+import org.apache.spark.ml.util.MLTestingUtils
+import org.apache.spark.mllib.classification.LogisticRegressionSuite._
+import org.apache.spark.mllib.linalg.{BLAS, DenseVector, Vectors}
+import org.apache.spark.mllib.random._
+import org.apache.spark.mllib.regression.LabeledPoint
+import org.apache.spark.mllib.util.MLlibTestSparkContext
+import org.apache.spark.mllib.util.TestingUtils._
+import org.apache.spark.sql.{DataFrame, Row}
+
+class GeneralizedLinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
+
+ private val seed: Int = 42
+ @transient var datasetGaussianIdentity: DataFrame = _
+ @transient var datasetGaussianLog: DataFrame = _
+ @transient var datasetGaussianInverse: DataFrame = _
+ @transient var datasetBinomial: DataFrame = _
+ @transient var datasetPoissonLog: DataFrame = _
+ @transient var datasetPoissonIdentity: DataFrame = _
+ @transient var datasetPoissonSqrt: DataFrame = _
+ @transient var datasetGammaInverse: DataFrame = _
+ @transient var datasetGammaIdentity: DataFrame = _
+ @transient var datasetGammaLog: DataFrame = _
+
+ override def beforeAll(): Unit = {
+ super.beforeAll()
+
+ import GeneralizedLinearRegressionSuite._
+
+ datasetGaussianIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "identity"), 2))
+
+ datasetGaussianLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "log"), 2))
+
+ datasetGaussianInverse = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "inverse"), 2))
+
+ datasetBinomial = {
+ val nPoints = 10000
+ val coefficients = Array(-0.57997, 0.912083, -0.371077, -0.819866, 2.688191)
+ val xMean = Array(5.843, 3.057, 3.758, 1.199)
+ val xVariance = Array(0.6856, 0.1899, 3.116, 0.581)
+
+ val testData =
+ generateMultinomialLogisticInput(coefficients, xMean, xVariance, true, nPoints, seed)
+
+ sqlContext.createDataFrame(sc.parallelize(testData, 4))
+ }
+
+ datasetPoissonLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "log"), 2))
+
+ datasetPoissonIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "identity"), 2))
+
+ datasetPoissonSqrt = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "sqrt"), 2))
+
+ datasetGammaInverse = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "inverse"), 2))
+
+ datasetGammaIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "identity"), 2))
+
+ datasetGammaLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "log"), 2))
+ }
+
+ test("params") {
+ ParamsSuite.checkParams(new GeneralizedLinearRegression)
+ val model = new GeneralizedLinearRegressionModel("genLinReg", Vectors.dense(0.0), 0.0)
+ ParamsSuite.checkParams(model)
+ }
+
+ test("generalized linear regression: default params") {
+ val glr = new GeneralizedLinearRegression
+ assert(glr.getLabelCol === "label")
+ assert(glr.getFeaturesCol === "features")
+ assert(glr.getPredictionCol === "prediction")
+ assert(glr.getFitIntercept)
+ assert(glr.getTol === 1E-6)
+ assert(glr.getWeightCol === "")
+ assert(glr.getRegParam === 0.0)
+ assert(glr.getSolver == "irls")
+ val model = glr.setFamily("gaussian").setLink("identity")
+ .fit(datasetGaussianIdentity)
+
+ // copied model must have the same parent.
+ MLTestingUtils.checkCopy(model)
+ model.transform(datasetGaussianIdentity)
+ .select("label", "prediction")
+ .collect()
+
+ assert(model.getFeaturesCol === "features")
+ assert(model.getPredictionCol === "prediction")
+ assert(model.intercept !== 0.0)
+ assert(model.hasParent)
+ assert(model.getFamily === "gaussian")
+ assert(model.getLink === "identity")
+ }
+
+ test("generalized linear regression: gaussian family against glm") {
+ /*
+ R code:
+ f1 <- data$V1 ~ data$V2 + data$V3 - 1
+ f2 <- data$V1 ~ data$V2 + data$V3
+
+ data <- read.csv("path", header=FALSE)
--- End diff --
Good point! I will send another PR after this get merged. Other test cases such as ```LinearRegressionSuite```, ```LogisticRegressionSuite``` and ```AFTSurvivalRegressionSuite``` can also benefit from this feature.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r53722634
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,547 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ if (isDefined(link)) {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
+ s"Generalized Linear Regression with ${$(family)} family does not support ${$(link)} " +
+ s"link function.")
+ }
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor and
+ * a description of the error distribution.
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("genLinReg"))
+
+ /**
+ * Set the name of family which is a description of the error distribution
+ * to be used in the model.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+
+ /**
+ * Set the name of the model link function.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Set if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+ setDefault(fitIntercept -> true)
+
+ /**
+ * Set the maximum number of iterations.
+ * Default is 100.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+ setDefault(maxIter -> 100)
+
+ /**
+ * Set the convergence tolerance of iterations.
+ * Smaller value will lead to higher accuracy with the cost of more iterations.
+ * Default is 1E-6.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setTol(value: Double): this.type = set(tol, value)
+ setDefault(tol -> 1E-6)
+
+ /**
+ * Set the regularization parameter.
+ * Default is 0.0.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setRegParam(value: Double): this.type = set(regParam, value)
+ setDefault(regParam -> 0.0)
+
+ /**
+ * Whether to over-/under-sample training instances according to the given weights in weightCol.
+ * If empty, all instances are treated equally (weight 1.0).
+ * Default is empty, so all instances have weight one.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setWeightCol(value: String): this.type = set(weightCol, value)
+ setDefault(weightCol -> "")
+
+ /**
+ * Set the solver algorithm used for optimization.
+ * Currently only support "irls" which is also the default solver.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setSolver(value: String): this.type = set(solver, value)
+ setDefault(solver -> "irls")
+
+ override protected def train(dataset: DataFrame): GeneralizedLinearRegressionModel = {
+ val familyLink = $(family) match {
+ case "gaussian" => if (isDefined(link)) Gaussian($(link)) else Gaussian("identity")
+ case "binomial" => if (isDefined(link)) Binomial($(link)) else Binomial("logit")
+ case "poisson" => if (isDefined(link)) Poisson($(link)) else Poisson("log")
+ case "gamma" => if (isDefined(link)) Gamma($(link)) else Gamma("inverse")
+ }
+
+ val numFeatures = dataset.select(col($(featuresCol))).limit(1).map {
+ case Row(features: Vector) => features.size
+ }.first()
+ if (numFeatures > 4096) {
+ val msg = "Currently, GeneralizedLinearRegression only supports number of features" +
+ s" <= 4096. Found $numFeatures in the input dataset."
+ throw new SparkException(msg)
+ }
+
+ val w = if ($(weightCol).isEmpty) lit(1.0) else col($(weightCol))
+ val instances: RDD[Instance] = dataset.select(
+ col($(labelCol)), w, col($(featuresCol))).map {
+ case Row(label: Double, weight: Double, features: Vector) =>
+ Instance(label, weight, features)
+ }
+
+ if ($(family) == "gaussian" && $(link) == "identity") {
+ val optimizer = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true)
+ val wlsModel = optimizer.fit(instances)
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ wlsModel.coefficients, wlsModel.intercept).setParent(this))
+ return model
+ }
+
+ val newInstances = instances.map { instance =>
+ val mu = familyLink.initialize(instance.label, instance.weight)
+ val eta = familyLink.predict(mu)
+ Instance(eta, instance.weight, instance.features)
+ }
+
+ val initialModel = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true).fit(newInstances)
+
+ val reweightFunc: (Instance, WeightedLeastSquaresModel) => (Double, Double) = {
+ (instance: Instance, model: WeightedLeastSquaresModel) => {
+ val eta = model.predict(instance.features)
+ val mu = familyLink.fitted(eta)
+ val z = familyLink.adjusted(instance.label, mu, eta)
+ val w = familyLink.weights(mu) * instance.weight
+ (z, w)
+ }
+ }
+
+ val optimizer = new IterativelyReweightedLeastSquares(initialModel, reweightFunc,
+ $(fitIntercept), $(regParam), $(maxIter), $(tol))
+
+ val irlsModel = optimizer.fit(instances)
+
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ irlsModel.coefficients, irlsModel.intercept).setParent(this))
--- End diff --
ditto on code alignment
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by yanboliang <gi...@git.apache.org>.
Github user yanboliang commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-190526311
Jenkins, test this please.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by AmplabJenkins <gi...@git.apache.org>.
Github user AmplabJenkins commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-188138718
Test PASSed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/51857/
Test PASSed.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-190530637
**[Test build #52215 has finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/52215/consoleFull)** for PR 11136 at commit [`314b562`](https://github.com/apache/spark/commit/314b562f315723a7117851289c8f5b6e1b16a6ac).
* This patch **fails to build**.
* This patch merges cleanly.
* This patch adds no public classes.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by AmplabJenkins <gi...@git.apache.org>.
Github user AmplabJenkins commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-182935182
Merged build finished. Test FAILed.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by yanboliang <gi...@git.apache.org>.
Github user yanboliang commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r54519291
--- Diff: mllib/src/test/scala/org/apache/spark/ml/regression/GeneralizedLinearRegressionSuite.scala ---
@@ -0,0 +1,499 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import scala.util.Random
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.ml.param.ParamsSuite
+import org.apache.spark.ml.util.MLTestingUtils
+import org.apache.spark.mllib.classification.LogisticRegressionSuite._
+import org.apache.spark.mllib.linalg.{BLAS, DenseVector, Vectors}
+import org.apache.spark.mllib.random._
+import org.apache.spark.mllib.regression.LabeledPoint
+import org.apache.spark.mllib.util.MLlibTestSparkContext
+import org.apache.spark.mllib.util.TestingUtils._
+import org.apache.spark.sql.{DataFrame, Row}
+
+class GeneralizedLinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
+
+ private val seed: Int = 42
+ @transient var datasetGaussianIdentity: DataFrame = _
+ @transient var datasetGaussianLog: DataFrame = _
+ @transient var datasetGaussianInverse: DataFrame = _
+ @transient var datasetBinomial: DataFrame = _
+ @transient var datasetPoissonLog: DataFrame = _
+ @transient var datasetPoissonIdentity: DataFrame = _
+ @transient var datasetPoissonSqrt: DataFrame = _
+ @transient var datasetGammaInverse: DataFrame = _
+ @transient var datasetGammaIdentity: DataFrame = _
+ @transient var datasetGammaLog: DataFrame = _
+
+ override def beforeAll(): Unit = {
+ super.beforeAll()
+
+ import GeneralizedLinearRegressionSuite._
+
+ datasetGaussianIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "identity"), 2))
+
+ datasetGaussianLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "log"), 2))
+
+ datasetGaussianInverse = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "inverse"), 2))
+
+ datasetBinomial = {
+ val nPoints = 10000
+ val coefficients = Array(-0.57997, 0.912083, -0.371077, -0.819866, 2.688191)
+ val xMean = Array(5.843, 3.057, 3.758, 1.199)
+ val xVariance = Array(0.6856, 0.1899, 3.116, 0.581)
+
+ val testData =
+ generateMultinomialLogisticInput(coefficients, xMean, xVariance, true, nPoints, seed)
+
+ sqlContext.createDataFrame(sc.parallelize(testData, 4))
+ }
+
+ datasetPoissonLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "log"), 2))
+
+ datasetPoissonIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "identity"), 2))
+
+ datasetPoissonSqrt = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "sqrt"), 2))
+
+ datasetGammaInverse = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "inverse"), 2))
+
+ datasetGammaIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "identity"), 2))
+
+ datasetGammaLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "log"), 2))
+ }
+
+ test("params") {
+ ParamsSuite.checkParams(new GeneralizedLinearRegression)
+ val model = new GeneralizedLinearRegressionModel("genLinReg", Vectors.dense(0.0), 0.0)
+ ParamsSuite.checkParams(model)
+ }
+
+ test("generalized linear regression: default params") {
+ val glr = new GeneralizedLinearRegression
+ assert(glr.getLabelCol === "label")
+ assert(glr.getFeaturesCol === "features")
+ assert(glr.getPredictionCol === "prediction")
+ assert(glr.getFitIntercept)
+ assert(glr.getTol === 1E-6)
+ assert(glr.getWeightCol === "")
+ assert(glr.getRegParam === 0.0)
+ assert(glr.getSolver == "irls")
+ val model = glr.setFamily("gaussian").setLink("identity")
+ .fit(datasetGaussianIdentity)
+
+ // copied model must have the same parent.
+ MLTestingUtils.checkCopy(model)
+ model.transform(datasetGaussianIdentity)
+ .select("label", "prediction")
+ .collect()
+
+ assert(model.getFeaturesCol === "features")
+ assert(model.getPredictionCol === "prediction")
+ assert(model.intercept !== 0.0)
+ assert(model.hasParent)
+ assert(model.getFamily === "gaussian")
+ assert(model.getLink === "identity")
+ }
+
+ test("generalized linear regression: gaussian family against glm") {
+ /*
+ R code:
+ f1 <- data$V1 ~ data$V2 + data$V3 - 1
+ f2 <- data$V1 ~ data$V2 + data$V3
+
+ data <- read.csv("path", header=FALSE)
+ for (formula in c(f1, f2)) {
+ model <- glm(formula, family="gaussian", data=data)
+ print(as.vector(coef(model)))
+ }
+
+ [1] 2.2960999 0.8087933
+ [1] 2.5002642 2.2000403 0.5999485
+
+ data <- read.csv("path", header=FALSE)
+ model1 <- glm(f1, family=gaussian(link=log), data=data, start=c(0,0))
+ model2 <- glm(f2, family=gaussian(link=log), data=data, start=c(0,0,0))
+ print(as.vector(coef(model1)))
+ print(as.vector(coef(model2)))
+
+ [1] 0.23069326 0.07993778
+ [1] 0.25001858 0.22002452 0.05998789
+
+ data <- read.csv("path", header=FALSE)
+ for (formula in c(f1, f2)) {
+ model <- glm(formula, family=gaussian(link=inverse), data=data)
+ print(as.vector(coef(model)))
+ }
+
+ [1] 2.3010179 0.8198976
+ [1] 2.4108902 2.2130248 0.6086152
+ */
+
+ val expected = Seq(
+ Vectors.dense(0.0, 2.2960999, 0.8087933),
+ Vectors.dense(2.5002642, 2.2000403, 0.5999485),
+ Vectors.dense(0.0, 0.23069326, 0.07993778),
+ Vectors.dense(0.25001858, 0.22002452, 0.05998789),
+ Vectors.dense(0.0, 2.3010179, 0.8198976),
+ Vectors.dense(2.4108902, 2.2130248, 0.6086152))
+
+ import GeneralizedLinearRegression._
+
+ var idx = 0
+ for ((link, dataset) <- Seq(("identity", datasetGaussianIdentity), ("log", datasetGaussianLog),
+ ("inverse", datasetGaussianInverse))) {
+ for (fitIntercept <- Seq(false, true)) {
+ val trainer = new GeneralizedLinearRegression().setFamily("gaussian").setLink(link)
+ .setFitIntercept(fitIntercept)
+ val model = trainer.fit(dataset)
+ val actual = Vectors.dense(model.intercept, model.coefficients(0), model.coefficients(1))
+ assert(actual ~== expected(idx) absTol 1e-4)
+
+ val familyLink = new FamilyAndLink(Gaussian, Link.fromName(link))
+ model.transform(dataset).select("features", "prediction").collect().foreach {
+ case Row(features: DenseVector, prediction1: Double) =>
+ val eta = BLAS.dot(features, model.coefficients) + model.intercept
+ val prediction2 = familyLink.fitted(eta)
+ assert(prediction1 ~== prediction2 relTol 1E-5)
+ }
+
+ idx += 1
+ }
+ }
+ }
+
+ test("generalized linear regression: gaussian family against glmnet") {
+ /*
+ R code:
+ library(glmnet)
+ data <- read.csv("path", header=FALSE)
+ label = data$V1
+ features = as.matrix(data.frame(data$V2, data$V3))
+ for (intercept in c(FALSE, TRUE)) {
+ for (lambda in c(0.0, 0.1, 1.0)) {
+ model <- glmnet(features, label, family="gaussian", intercept=intercept,
+ lambda=lambda, alpha=0, thresh=1E-14)
+ print(as.vector(coef(model)))
+ }
+ }
+
+ [1] 0.0000000 2.2961005 0.8087932
+ [1] 0.0000000 2.2130368 0.8309556
+ [1] 0.0000000 1.7176137 0.9610657
+ [1] 2.5002642 2.2000403 0.5999485
+ [1] 3.1106389 2.0935142 0.5712711
+ [1] 6.7597127 1.4581054 0.3994266
+ */
+
+ val expected = Seq(
+ Vectors.dense(0.0, 2.2961005, 0.8087932),
+ Vectors.dense(0.0, 2.2130368, 0.8309556),
+ Vectors.dense(0.0, 1.7176137, 0.9610657),
+ Vectors.dense(2.5002642, 2.2000403, 0.5999485),
+ Vectors.dense(3.1106389, 2.0935142, 0.5712711),
+ Vectors.dense(6.7597127, 1.4581054, 0.3994266))
+
+ var idx = 0
+ for (fitIntercept <- Seq(false, true);
+ regParam <- Seq(0.0, 0.1, 1.0)) {
+ val trainer = new GeneralizedLinearRegression().setFamily("gaussian")
+ .setFitIntercept(fitIntercept).setRegParam(regParam)
+ val model = trainer.fit(datasetGaussianIdentity)
+ val actual = Vectors.dense(model.intercept, model.coefficients(0), model.coefficients(1))
+ assert(actual ~== expected(idx) absTol 1e-4)
+
+ idx += 1
+ }
+ }
+
+ test("generalized linear regression: binomial family against glm") {
+ /*
+ R code:
+ f1 <- data$V1 ~ data$V2 + data$V3 + data$V4 + data$V5 - 1
+ f2 <- data$V1 ~ data$V2 + data$V3 + data$V4 + data$V5
+ data <- read.csv("path", header=FALSE)
+
+ for (formula in c(f1, f2)) {
+ model <- glm(formula, family="binomial", data=data)
+ print(as.vector(coef(model)))
+ }
+
+ [1] -0.3560284 1.3010002 -0.3570805 -0.7406762
+ [1] 2.8367406 -0.5896187 0.8931655 -0.3925169 -0.7996989
+
+ for (formula in c(f1, f2)) {
+ model <- glm(formula, family=binomial(link=probit), data=data)
+ print(as.vector(coef(model)))
+ }
+
+ [1] -0.2134390 0.7800646 -0.2144267 -0.4438358
+ [1] 1.6995366 -0.3524694 0.5332651 -0.2352985 -0.4780850
+
+ for (formula in c(f1, f2)) {
+ model <- glm(formula, family=binomial(link=cloglog), data=data)
+ print(as.vector(coef(model)))
+ }
+
+ [1] -0.2832198 0.8434144 -0.2524727 -0.5293452
+ [1] 1.5063590 -0.4038015 0.6133664 -0.2687882 -0.5541758
+ */
+ val expected = Seq(
+ Vectors.dense(0.0, -0.3560284, 1.3010002, -0.3570805, -0.7406762),
+ Vectors.dense(2.8367406, -0.5896187, 0.8931655, -0.3925169, -0.7996989),
+ Vectors.dense(0.0, -0.2134390, 0.7800646, -0.2144267, -0.4438358),
+ Vectors.dense(1.6995366, -0.3524694, 0.5332651, -0.2352985, -0.4780850),
+ Vectors.dense(0.0, -0.2832198, 0.8434144, -0.2524727, -0.5293452),
+ Vectors.dense(1.5063590, -0.4038015, 0.6133664, -0.2687882, -0.5541758))
+
+ import GeneralizedLinearRegression._
+
+ var idx = 0
+ for ((link, dataset) <- Seq(("logit", datasetBinomial), ("probit", datasetBinomial),
+ ("cloglog", datasetBinomial))) {
+ for (fitIntercept <- Seq(false, true)) {
+ val trainer = new GeneralizedLinearRegression().setFamily("binomial").setLink(link)
+ .setFitIntercept(fitIntercept)
+ val model = trainer.fit(dataset)
+ val actual = Vectors.dense(model.intercept, model.coefficients(0), model.coefficients(1),
+ model.coefficients(2), model.coefficients(3))
+ assert(actual ~== expected(idx) absTol 1e-4)
+
+ val familyLink = new FamilyAndLink(Binomial, Link.fromName(link))
+ model.transform(dataset).select("features", "prediction").collect().foreach {
+ case Row(features: DenseVector, prediction1: Double) =>
+ val eta = BLAS.dot(features, model.coefficients) + model.intercept
+ val prediction2 = familyLink.fitted(eta)
+ assert(prediction1 ~== prediction2 relTol 1E-5)
+ }
+
+ idx += 1
+ }
+ }
+ }
+
+ test("generalized linear regression: poisson family against glm") {
+ /*
+ R code:
+ f1 <- data$V1 ~ data$V2 + data$V3 - 1
+ f2 <- data$V1 ~ data$V2 + data$V3
+
+ data <- read.csv("path", header=FALSE)
+ for (formula in c(f1, f2)) {
+ model <- glm(formula, family="poisson", data=data)
+ print(as.vector(coef(model)))
+ }
+
+ [1] 0.22999393 0.08047088
+ [1] 0.25022353 0.21998599 0.05998621
+
+ data <- read.csv("path", header=FALSE)
+ for (formula in c(f1, f2)) {
+ model <- glm(formula, family=poisson(link=identity), data=data)
+ print(as.vector(coef(model)))
+ }
+
+ [1] 2.2929501 0.8119415
+ [1] 2.5012730 2.1999407 0.5999107
+
+ data <- read.csv("path", header=FALSE)
+ for (formula in c(f1, f2)) {
+ model <- glm(formula, family=poisson(link=sqrt), data=data)
+ print(as.vector(coef(model)))
+ }
+
+ [1] 2.2958947 0.8090515
+ [1] 2.5000480 2.1999972 0.5999968
+ */
+ val expected = Seq(
+ Vectors.dense(0.0, 0.22999393, 0.08047088),
+ Vectors.dense(0.25022353, 0.21998599, 0.05998621),
+ Vectors.dense(0.0, 2.2929501, 0.8119415),
+ Vectors.dense(2.5012730, 2.1999407, 0.5999107),
+ Vectors.dense(0.0, 2.2958947, 0.8090515),
+ Vectors.dense(2.5000480, 2.1999972, 0.5999968))
+
+ import GeneralizedLinearRegression._
+
+ var idx = 0
+ for ((link, dataset) <- Seq(("log", datasetPoissonLog), ("identity", datasetPoissonIdentity),
+ ("sqrt", datasetPoissonSqrt))) {
+ for (fitIntercept <- Seq(false, true)) {
+ val trainer = new GeneralizedLinearRegression().setFamily("poisson").setLink(link)
+ .setFitIntercept(fitIntercept)
+ val model = trainer.fit(dataset)
+ val actual = Vectors.dense(model.intercept, model.coefficients(0), model.coefficients(1))
+ assert(actual ~== expected(idx) absTol 1e-4)
+
+ val familyLink = new FamilyAndLink(Poisson, Link.fromName(link))
+ model.transform(dataset).select("features", "prediction").collect().foreach {
+ case Row(features: DenseVector, prediction1: Double) =>
+ val eta = BLAS.dot(features, model.coefficients) + model.intercept
+ val prediction2 = familyLink.fitted(eta)
+ assert(prediction1 ~== prediction2 relTol 1E-5)
+ }
+
+ idx += 1
+ }
+ }
+ }
+
+ test("generalized linear regression: gamma family against glm") {
+ /*
+ R code:
+ f1 <- data$V1 ~ data$V2 + data$V3 - 1
+ f2 <- data$V1 ~ data$V2 + data$V3
+
+ data <- read.csv("path", header=FALSE)
+ for (formula in c(f1, f2)) {
+ model <- glm(formula, family="Gamma", data=data)
+ print(as.vector(coef(model)))
+ }
+
+ [1] 2.3392419 0.8058058
+ [1] 2.3507700 2.2533574 0.6042991
+
+ data <- read.csv("path", header=FALSE)
+ for (formula in c(f1, f2)) {
+ model <- glm(formula, family=Gamma(link=identity), data=data)
+ print(as.vector(coef(model)))
+ }
+
+ [1] 2.2908883 0.8147796
+ [1] 2.5002406 2.1998346 0.6000059
+
+ data <- read.csv("path", header=FALSE)
+ for (formula in c(f1, f2)) {
+ model <- glm(formula, family=Gamma(link=log), data=data)
+ print(as.vector(coef(model)))
+ }
+
+ [1] 0.22958970 0.08091066
+ [1] 0.25003210 0.21996957 0.06000215
+ */
+ val expected = Seq(
+ Vectors.dense(0.0, 2.3392419, 0.8058058),
+ Vectors.dense(2.3507700, 2.2533574, 0.6042991),
+ Vectors.dense(0.0, 2.2908883, 0.8147796),
+ Vectors.dense(2.5002406, 2.1998346, 0.6000059),
+ Vectors.dense(0.0, 0.22958970, 0.08091066),
+ Vectors.dense(0.25003210, 0.21996957, 0.06000215))
+
+ import GeneralizedLinearRegression._
+
+ var idx = 0
+ for ((link, dataset) <- Seq(("inverse", datasetGammaInverse),
+ ("identity", datasetGammaIdentity), ("log", datasetGammaLog))) {
+ for (fitIntercept <- Seq(false, true)) {
+ val trainer = new GeneralizedLinearRegression().setFamily("gamma").setLink(link)
+ .setFitIntercept(fitIntercept)
+ val model = trainer.fit(dataset)
+ val actual = Vectors.dense(model.intercept, model.coefficients(0), model.coefficients(1))
+ assert(actual ~== expected(idx) absTol 1e-4)
+
+ val familyLink = new FamilyAndLink(Gamma, Link.fromName(link))
+ model.transform(dataset).select("features", "prediction").collect().foreach {
+ case Row(features: DenseVector, prediction1: Double) =>
+ val eta = BLAS.dot(features, model.coefficients) + model.intercept
+ val prediction2 = familyLink.fitted(eta)
+ assert(prediction1 ~== prediction2 relTol 1E-5)
+ }
+
+ idx += 1
+ }
+ }
+ }
+}
+
+object GeneralizedLinearRegressionSuite {
+
+ def generateGeneralizedLinearRegressionInput(
--- End diff --
Fixed.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r53722712
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,547 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ if (isDefined(link)) {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
+ s"Generalized Linear Regression with ${$(family)} family does not support ${$(link)} " +
+ s"link function.")
+ }
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor and
+ * a description of the error distribution.
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("genLinReg"))
+
+ /**
+ * Set the name of family which is a description of the error distribution
+ * to be used in the model.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+
+ /**
+ * Set the name of the model link function.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Set if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+ setDefault(fitIntercept -> true)
+
+ /**
+ * Set the maximum number of iterations.
+ * Default is 100.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+ setDefault(maxIter -> 100)
+
+ /**
+ * Set the convergence tolerance of iterations.
+ * Smaller value will lead to higher accuracy with the cost of more iterations.
+ * Default is 1E-6.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setTol(value: Double): this.type = set(tol, value)
+ setDefault(tol -> 1E-6)
+
+ /**
+ * Set the regularization parameter.
+ * Default is 0.0.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setRegParam(value: Double): this.type = set(regParam, value)
+ setDefault(regParam -> 0.0)
+
+ /**
+ * Whether to over-/under-sample training instances according to the given weights in weightCol.
+ * If empty, all instances are treated equally (weight 1.0).
+ * Default is empty, so all instances have weight one.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setWeightCol(value: String): this.type = set(weightCol, value)
+ setDefault(weightCol -> "")
+
+ /**
+ * Set the solver algorithm used for optimization.
+ * Currently only support "irls" which is also the default solver.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setSolver(value: String): this.type = set(solver, value)
+ setDefault(solver -> "irls")
+
+ override protected def train(dataset: DataFrame): GeneralizedLinearRegressionModel = {
+ val familyLink = $(family) match {
+ case "gaussian" => if (isDefined(link)) Gaussian($(link)) else Gaussian("identity")
+ case "binomial" => if (isDefined(link)) Binomial($(link)) else Binomial("logit")
+ case "poisson" => if (isDefined(link)) Poisson($(link)) else Poisson("log")
+ case "gamma" => if (isDefined(link)) Gamma($(link)) else Gamma("inverse")
+ }
+
+ val numFeatures = dataset.select(col($(featuresCol))).limit(1).map {
+ case Row(features: Vector) => features.size
+ }.first()
+ if (numFeatures > 4096) {
+ val msg = "Currently, GeneralizedLinearRegression only supports number of features" +
+ s" <= 4096. Found $numFeatures in the input dataset."
+ throw new SparkException(msg)
+ }
+
+ val w = if ($(weightCol).isEmpty) lit(1.0) else col($(weightCol))
+ val instances: RDD[Instance] = dataset.select(
+ col($(labelCol)), w, col($(featuresCol))).map {
+ case Row(label: Double, weight: Double, features: Vector) =>
+ Instance(label, weight, features)
+ }
+
+ if ($(family) == "gaussian" && $(link) == "identity") {
+ val optimizer = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true)
+ val wlsModel = optimizer.fit(instances)
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ wlsModel.coefficients, wlsModel.intercept).setParent(this))
+ return model
+ }
+
+ val newInstances = instances.map { instance =>
+ val mu = familyLink.initialize(instance.label, instance.weight)
+ val eta = familyLink.predict(mu)
+ Instance(eta, instance.weight, instance.features)
+ }
+
+ val initialModel = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true).fit(newInstances)
+
+ val reweightFunc: (Instance, WeightedLeastSquaresModel) => (Double, Double) = {
+ (instance: Instance, model: WeightedLeastSquaresModel) => {
+ val eta = model.predict(instance.features)
+ val mu = familyLink.fitted(eta)
+ val z = familyLink.adjusted(instance.label, mu, eta)
+ val w = familyLink.weights(mu) * instance.weight
+ (z, w)
+ }
+ }
+
+ val optimizer = new IterativelyReweightedLeastSquares(initialModel, reweightFunc,
+ $(fitIntercept), $(regParam), $(maxIter), $(tol))
+
+ val irlsModel = optimizer.fit(instances)
+
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ irlsModel.coefficients, irlsModel.intercept).setParent(this))
+ model
+ }
+
+ @Since("2.0.0")
+ override def copy(extra: ParamMap): GeneralizedLinearRegression = defaultCopy(extra)
+}
+
+@Since("2.0.0")
+object GeneralizedLinearRegression {
+
+ /** Set of families that GeneralizedLinearRegression supports */
+ private[ml] val supportedFamilies = Set("gaussian", "binomial", "poisson", "gamma")
+
+ /** Set of links that GeneralizedLinearRegression supports */
+ private[ml] val supportedLinks = Set("identity", "log", "inverse", "logit", "probit",
+ "cloglog", "sqrt")
+
+ /** Set of family and link pairs that GeneralizedLinearRegression supports */
+ private[ml] val supportedFamilyLinkPairs = Set(
+ "gaussian" -> "identity", "gaussian" -> "log", "gaussian" -> "inverse",
+ "binomial" -> "logit", "binomial" -> "probit", "binomial" -> "cloglog",
+ "poisson" -> "log", "poisson" -> "identity", "poisson" -> "sqrt",
+ "gamma" -> "inverse", "gamma" -> "identity", "gamma" -> "log"
+ )
+}
+
+/**
+ * :: Experimental ::
+ * Model produced by [[GeneralizedLinearRegression]].
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegressionModel private[ml] (
+ @Since("2.0.0") override val uid: String,
+ @Since("2.0.0") val coefficients: Vector,
+ @Since("2.0.0") val intercept: Double)
+ extends RegressionModel[Vector, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams {
+
+ private lazy val familyLink = $(family) match {
+ case "gaussian" => if (isDefined(link)) Gaussian($(link)) else Gaussian("identity")
+ case "binomial" => if (isDefined(link)) Binomial($(link)) else Binomial("logit")
+ case "poisson" => if (isDefined(link)) Poisson($(link)) else Poisson("log")
+ case "gamma" => if (isDefined(link)) Gamma($(link)) else Gamma("inverse")
+ }
+
+ override protected def predict(features: Vector): Double = {
+ val eta = BLAS.dot(features, coefficients) + intercept
+ familyLink.fitted(eta)
+ }
+
+ @Since("2.0.0")
+ override def copy(extra: ParamMap): GeneralizedLinearRegressionModel = {
+ copyValues(new GeneralizedLinearRegressionModel(uid, coefficients, intercept), extra)
+ .setParent(parent)
+ }
+}
+
+/**
+ * A description of the error distribution and link function to be used in the model.
+ * @param link a link function instance
+ */
+private[ml] abstract class Family(val link: Link) extends Serializable {
+
+ /** Initialize the starting value for mu. */
+ def initialize(y: Double, weight: Double): Double
+
+ /** The variance of the endogenous variable's mean, given the value mu. */
+ def variance(mu: Double): Double
+
+ /** Weights for IRLS steps. */
+ def weights(mu: Double): Double = {
+ val x = clean(mu)
+ 1.0 / (math.pow(this.link.deriv(x), 2.0) * this.variance(x))
+ }
+
+ /** The adjusted response variable. */
+ def adjusted(y: Double, mu: Double, eta: Double): Double = {
+ val x = clean(mu)
+ eta + (y - x) * link.deriv(x)
+ }
+
+ /** Linear predictor based on given mu. */
+ def predict(mu: Double): Double = this.link.link(clean(mu))
+
+ /** Fitted value based on linear predictor eta. */
+ def fitted(eta: Double): Double = clean(this.link.unlink(eta))
+
+ /** Trim the fitted value so that it will be in valid range. */
+ def clean(mu: Double): Double = mu
+
+ val epsilon: Double = 1E-16
+}
+
+/**
+ * Gaussian exponential family distribution.
+ * The default link for the Gaussian family is the identity link.
+ * @param link a link function instance
+ */
+private[ml] class Gaussian(link: Link = Identity) extends Family(link) {
+
+ override def initialize(y: Double, weight: Double): Double = {
+ if (link == Log) {
+ require(y > 0.0, "The response variable of Gaussian family with Log link " +
+ s"should be positive, but got $y")
+ }
+ y
+ }
+
+ def variance(mu: Double): Double = 1.0
+
+ override def clean(mu: Double): Double = {
+ if (mu.isNegInfinity) {
+ Double.MinValue
+ } else if (mu.isPosInfinity) {
+ Double.MaxValue
+ } else {
+ mu
+ }
+ }
+}
+
+private[ml] object Gaussian {
+
+ def apply(link: String): Gaussian = {
+ link match {
+ case "identity" => new Gaussian(Identity)
+ case "log" => new Gaussian(Log)
+ case "inverse" => new Gaussian(Inverse)
+ }
+ }
+}
+
+/**
+ * Binomial exponential family distribution.
+ * The default link for the Binomial family is the logit link.
+ * @param link a link function instance
+ */
+private[ml] class Binomial(link: Link = Logit) extends Family(link) {
+
+ override def initialize(y: Double, weight: Double): Double = {
+ val mu = (weight * y + 0.5) / (weight + 1.0)
+ require(mu > 0.0 && mu < 1.0, "The response variable of Binomial family" +
+ s"should be in range (0, 1), but got $mu")
+ mu
+ }
+
+ override def variance(mu: Double): Double = mu * (1.0 - mu)
+
+ override def clean(mu: Double): Double = {
+ if (mu < epsilon) {
+ epsilon
+ } else if (mu > 1.0 - epsilon) {
+ 1.0 - epsilon
+ } else {
+ mu
+ }
+ }
+}
+
+private[ml] object Binomial {
+
+ def apply(link: String): Binomial = {
+ link match {
+ case "logit" => new Binomial(Logit)
+ case "probit" => new Binomial(Probit)
+ case "cloglog" => new Binomial(CLogLog)
+ }
+ }
+}
+
+/**
+ * Poisson exponential family distribution.
+ * The default link for the Poisson family is the log link.
+ * @param link a link function instance
+ */
+private[ml] class Poisson(link: Link = Log) extends Family(link) {
+
+ override def initialize(y: Double, weight: Double): Double = {
+ require(y > 0.0, "The response variable of Poisson family " +
+ s"should be positive, but got $y")
+ y
+ }
+
+ override def variance(mu: Double): Double = mu
+
+ override def clean(mu: Double): Double = {
+ if (mu < epsilon) {
+ epsilon
+ } else if (mu.isInfinity) {
+ Double.MaxValue
+ } else {
+ mu
+ }
+ }
+}
+
+private[ml] object Poisson {
+
+ def apply(link: String): Poisson = {
+ link match {
+ case "log" => new Poisson(Log)
+ case "sqrt" => new Poisson(Sqrt)
+ case "identity" => new Poisson(Identity)
+ }
+ }
+}
+
+/**
+ * Gamma exponential family distribution.
+ * The default link for the Gamma family is the inverse link.
+ * @param link a link function instance
+ */
+private[ml] class Gamma(link: Link = Inverse) extends Family(link) {
+
+ override def initialize(y: Double, weight: Double): Double = {
+ require(y > 0.0, "The response variable of Gamma family " +
+ s"should be positive, but got $y")
+ y
+ }
+
+ override def variance(mu: Double): Double = math.pow(mu, 2.0)
+
+ override def clean(mu: Double): Double = {
+ if (mu < epsilon) {
+ epsilon
+ } else if (mu.isInfinity) {
+ Double.MaxValue
+ } else {
+ mu
+ }
+ }
+}
+
+private[ml] object Gamma {
+
+ def apply(link: String): Gamma = {
+ link match {
+ case "inverse" => new Gamma(Inverse)
+ case "identity" => new Gamma(Identity)
+ case "log" => new Gamma(Log)
+ }
+ }
+}
+
+/**
+ * A description of the link function to be used in the model.
+ */
+private[ml] trait Link extends Serializable {
--- End diff --
I would suggest moving `Link` to `object GeneralizedLinearRegression`.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r53722391
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,547 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
--- End diff --
* Include supported options and the default value in the param doc (and the ScalaDoc).
* Shall we make "gaussian" the default?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by AmplabJenkins <gi...@git.apache.org>.
Github user AmplabJenkins commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-183874428
Merged build finished. Test PASSed.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-189210653
**[Test build #52044 has finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/52044/consoleFull)** for PR 11136 at commit [`c05a948`](https://github.com/apache/spark/commit/c05a94899c39bfa9ede9071bd6db6a937b33cf83).
* This patch passes all tests.
* This patch merges cleanly.
* This patch adds no public classes.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by yanboliang <gi...@git.apache.org>.
Github user yanboliang commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r52331693
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,472 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.Logging
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
+ s"Generalized Linear Regression with ${$(family)} family does not support ${$(link)} " +
+ s"link function.")
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor and
+ * a description of the error distribution.
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("genLinReg"))
+
+ /**
+ * Set the name of family which is a description of the error distribution
+ * to be used in the model.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+
+ /**
+ * Set the name of the model link function.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Set if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+ setDefault(fitIntercept -> true)
+
+ /**
+ * Set the maximum number of iterations.
+ * Default is 100.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+ setDefault(maxIter -> 100)
+
+ /**
+ * Set the convergence tolerance of iterations.
+ * Smaller value will lead to higher accuracy with the cost of more iterations.
+ * Default is 1E-6.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setTol(value: Double): this.type = set(tol, value)
+ setDefault(tol -> 1E-6)
+
+ /**
+ * Set the regularization parameter.
+ * Default is 0.0.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setRegParam(value: Double): this.type = set(regParam, value)
+ setDefault(regParam -> 0.0)
+
+ /**
+ * Whether to over-/under-sample training instances according to the given weights in weightCol.
+ * If empty, all instances are treated equally (weight 1.0).
+ * Default is empty, so all instances have weight one.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setWeightCol(value: String): this.type = set(weightCol, value)
+ setDefault(weightCol -> "")
+
+ /**
+ * Set the solver algorithm used for optimization.
+ * Currently only support "irls" which is also the default solver.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setSolver(value: String): this.type = set(solver, value)
+ setDefault(solver -> "irls")
+
+ override protected def train(dataset: DataFrame): GeneralizedLinearRegressionModel = {
+ val familyLink = $(family) match {
+ case "gaussian" => Gaussian($(link))
+ case "binomial" => Binomial($(link))
+ case "poisson" => Poisson($(link))
+ case "gamma" => Gamma($(link))
+ }
+
+ val w = if ($(weightCol).isEmpty) lit(1.0) else col($(weightCol))
+ val instances: RDD[Instance] = dataset.select(
+ col($(labelCol)), w, col($(featuresCol))).map {
+ case Row(label: Double, weight: Double, features: Vector) =>
+ Instance(label, weight, features)
+ }
+
+ if ($(family) == "gaussian" && $(link) == "identity") {
+ val optimizer = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true)
+ val wlsModel = optimizer.fit(instances)
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ wlsModel.coefficients, wlsModel.intercept).setParent(this))
+ return model
+ }
+
+ val newInstances = instances.map { instance =>
+ val mu = familyLink.initialize(instance.label, instance.weight)
+ val eta = familyLink.predict(mu)
+ Instance(eta, instance.weight, instance.features)
+ }
+
+ val initialModel = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true).fit(newInstances)
+
+ val reweightFunc: (Instance, WeightedLeastSquaresModel) => (Double, Double) = {
+ (instance: Instance, model: WeightedLeastSquaresModel) => {
+ val eta = model.predict(instance.features)
+ val mu = familyLink.fitted(eta)
+ val z = familyLink.adjusted(instance.label, mu, eta)
+ val w = familyLink.weights(mu) * instance.weight
+ (z, w)
+ }
+ }
+
+ val optimizer = new IterativelyReweightedLeastSquares(initialModel, reweightFunc,
+ $(fitIntercept), $(regParam), $(maxIter), $(tol))
+
+ val irlsModel = optimizer.fit(instances)
+
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ irlsModel.coefficients, irlsModel.intercept).setParent(this))
+ model
+ }
+
+ @Since("2.0.0")
+ override def copy(extra: ParamMap): GeneralizedLinearRegression = defaultCopy(extra)
+}
+
+@Since("2.0.0")
+object GeneralizedLinearRegression {
+
+ /** Set of families that GeneralizedLinearRegression supports */
+ private[ml] val supportedFamilies = Set("gaussian", "binomial", "poisson", "gamma")
+
+ /** Set of links that GeneralizedLinearRegression supports */
+ private[ml] val supportedLinks = Set("identity", "log", "inverse", "logit", "probit",
+ "cloglog", "sqrt")
+
+ /** Set of family and link pairs that GeneralizedLinearRegression supports */
+ private[ml] val supportedFamilyLinkPairs = Set(
+ "gaussian" -> "identity", "gaussian" -> "log", "gaussian" -> "inverse",
+ "binomial" -> "logit", "binomial" -> "probit", "binomial" -> "cloglog",
+ "poisson" -> "log", "poisson" -> "identity", "poisson" -> "sqrt",
+ "gamma" -> "inverse", "gamma" -> "identity", "gamma" -> "log"
+ )
+}
+
+/**
+ * :: Experimental ::
+ * Model produced by [[GeneralizedLinearRegression]].
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegressionModel private[ml] (
+ @Since("2.0.0") override val uid: String,
+ @Since("2.0.0") val coefficients: Vector,
+ @Since("2.0.0") val intercept: Double)
+ extends RegressionModel[Vector, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams {
+
+ private lazy val familyLink = $(family) match {
+ case "gaussian" => Gaussian($(link))
+ case "binomial" => Binomial($(link))
+ case "poisson" => Poisson($(link))
+ case "gamma" => Gamma($(link))
+ }
+
+ override protected def predict(features: Vector): Double = {
+ val eta = BLAS.dot(features, coefficients) + intercept
+ familyLink.fitted(eta)
+ }
+
+ @Since("2.0.0")
+ override def copy(extra: ParamMap): GeneralizedLinearRegressionModel = {
+ copyValues(new GeneralizedLinearRegressionModel(uid, coefficients, intercept), extra)
+ .setParent(parent)
+ }
+}
+
+/**
+ * A description of the error distribution and link function to be used in the model.
+ * @param link a link function instance
+ */
+private[ml] abstract class Family(val link: Link) extends Serializable {
+
+ /** Initialize the starting value for mu. */
+ def initialize(y: Double, weight: Double): Double
+
+ /** The variance of mu to its mean. */
+ def variance(mu: Double): Double
+
+ /** Weights for IRLS steps. */
+ def weights(mu: Double): Double = {
+ 1.0 / (math.pow(this.link.deriv(mu), 2.0) * this.variance(mu))
+ }
+
+ /** The adjusted response variable. */
+ def adjusted(y: Double, mu: Double, eta: Double): Double = {
+ eta + (y - mu) * link.deriv(mu)
+ }
+
+ /** Linear predictors based on given mu. */
+ def predict(mu: Double): Double = this.link.link(mu)
+
+ /** Fitted values based on linear predictors eta. */
+ def fitted(eta: Double): Double = this.link.unlink(eta)
+}
+
+/**
+ * Gaussian exponential family distribution.
+ * The default link for the Gaussian family is the identity link.
+ * @param link a link function instance
+ */
+private[ml] class Gaussian(link: Link = Identity) extends Family(link) {
+
+ override def initialize(y: Double, weight: Double): Double = y
+
+ def variance(mu: Double): Double = 1.0
+}
+
+private[ml] object Gaussian {
+
+ def apply(link: String): Gaussian = {
+ link match {
+ case "identity" => new Gaussian(Identity)
+ case "log" => new Gaussian(Log)
+ case "inverse" => new Gaussian(Inverse)
+ }
+ }
+}
+
+/**
+ * Binomial exponential family distribution.
+ * The default link for the Binomial family is the logit link.
+ * @param link a link function instance
+ */
+private[ml] class Binomial(link: Link = Logit) extends Family(link) {
+
+ override def initialize(y: Double, weight: Double): Double = {
+ (weight * y + 0.5) / (weight + 1.0)
+ }
+
+ override def variance(mu: Double): Double = mu * (1 - mu)
+}
+
+private[ml] object Binomial {
+
+ def apply(link: String): Binomial = {
+ link match {
+ case "logit" => new Binomial(Logit)
+ case "probit" => new Binomial(Probit)
+ case "cloglog" => new Binomial(CLogLog)
+ }
+ }
+}
+
+/**
+ * Poisson exponential family distribution.
+ * The default link for the Poisson family is the log link.
+ * @param link a link function instance
+ */
+private[ml] class Poisson(link: Link = Log) extends Family(link) {
+
+ override def initialize(y: Double, weight: Double): Double = y + 0.1
+
+ override def variance(mu: Double): Double = mu
+}
+
+private[ml] object Poisson {
+
+ def apply(link: String): Poisson = {
+ link match {
+ case "log" => new Poisson(Log)
+ case "sqrt" => new Poisson(Sqrt)
+ case "identity" => new Poisson(Identity)
+ }
+ }
+}
+
+/**
+ * Gamma exponential family distribution.
+ * The default link for the Gamma family is the inverse link.
+ * @param link a link function instance
+ */
+private[ml] class Gamma(link: Link = Log) extends Family(link) {
+
+ override def initialize(y: Double, weight: Double): Double = y
+
+ override def variance(mu: Double): Double = math.pow(mu, 2.0)
+}
+
+private[ml] object Gamma {
+
+ def apply(link: String): Gamma = {
+ link match {
+ case "inverse" => new Gamma(Inverse)
+ case "identity" => new Gamma(Identity)
+ case "log" => new Gamma(Log)
+ }
+ }
+}
+
+/**
+ * A description of the link function to be used in the model.
+ */
+private[ml] trait Link extends Serializable {
+
+ /** The link function. */
+ def link(mu: Double): Double
+
+ /** The derivative function. */
+ def deriv(mu: Double): Double
+
+ /** The inverse link function. */
+ def unlink(eta: Double): Double
+}
+
+private[ml] object Identity extends Link {
+
+ override def link(mu: Double): Double = mu
+
+ override def deriv(mu: Double): Double = 1.0
+
+ override def unlink(eta: Double): Double = eta
+}
+
+private[ml] object Logit extends Link {
+
+ override def link(mu: Double): Double = math.log(mu / (1.0 - mu))
+
+ override def deriv(mu: Double): Double = 1.0 / (mu * (1.0 - mu))
+
+ override def unlink(eta: Double): Double = 1.0 / (1.0 + math.exp(-1.0 * eta))
+}
--- End diff --
The link functions should be further refinement to guarantee that the endogenous variable does not contain invalid values.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-189197945
**[Test build #52044 has started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/52044/consoleFull)** for PR 11136 at commit [`c05a948`](https://github.com/apache/spark/commit/c05a94899c39bfa9ede9071bd6db6a937b33cf83).
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-184174776
**[Test build #51306 has finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/51306/consoleFull)** for PR 11136 at commit [`4a27970`](https://github.com/apache/spark/commit/4a27970486091b6359b9d73ba477044d04c20875).
* This patch passes all tests.
* This patch merges cleanly.
* This patch adds no public classes.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by yanboliang <gi...@git.apache.org>.
Github user yanboliang commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r52330009
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,472 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.Logging
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
+ s"Generalized Linear Regression with ${$(family)} family does not support ${$(link)} " +
+ s"link function.")
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor and
+ * a description of the error distribution.
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("genLinReg"))
+
+ /**
+ * Set the name of family which is a description of the error distribution
+ * to be used in the model.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+
+ /**
+ * Set the name of the model link function.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Set if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+ setDefault(fitIntercept -> true)
+
+ /**
+ * Set the maximum number of iterations.
+ * Default is 100.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+ setDefault(maxIter -> 100)
+
+ /**
+ * Set the convergence tolerance of iterations.
+ * Smaller value will lead to higher accuracy with the cost of more iterations.
+ * Default is 1E-6.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setTol(value: Double): this.type = set(tol, value)
+ setDefault(tol -> 1E-6)
+
+ /**
+ * Set the regularization parameter.
+ * Default is 0.0.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setRegParam(value: Double): this.type = set(regParam, value)
+ setDefault(regParam -> 0.0)
+
+ /**
+ * Whether to over-/under-sample training instances according to the given weights in weightCol.
+ * If empty, all instances are treated equally (weight 1.0).
+ * Default is empty, so all instances have weight one.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setWeightCol(value: String): this.type = set(weightCol, value)
+ setDefault(weightCol -> "")
+
+ /**
+ * Set the solver algorithm used for optimization.
+ * Currently only support "irls" which is also the default solver.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setSolver(value: String): this.type = set(solver, value)
+ setDefault(solver -> "irls")
+
+ override protected def train(dataset: DataFrame): GeneralizedLinearRegressionModel = {
+ val familyLink = $(family) match {
+ case "gaussian" => Gaussian($(link))
+ case "binomial" => Binomial($(link))
+ case "poisson" => Poisson($(link))
+ case "gamma" => Gamma($(link))
+ }
+
+ val w = if ($(weightCol).isEmpty) lit(1.0) else col($(weightCol))
+ val instances: RDD[Instance] = dataset.select(
+ col($(labelCol)), w, col($(featuresCol))).map {
+ case Row(label: Double, weight: Double, features: Vector) =>
+ Instance(label, weight, features)
+ }
+
+ if ($(family) == "gaussian" && $(link) == "identity") {
--- End diff --
For ```gaussian``` family with ```identity``` link, we only use ```WeightedLeastSquares``` to train the model.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by AmplabJenkins <gi...@git.apache.org>.
Github user AmplabJenkins commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-190530641
Merged build finished. Test FAILed.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r54508443
--- Diff: mllib/src/test/scala/org/apache/spark/ml/regression/GeneralizedLinearRegressionSuite.scala ---
@@ -0,0 +1,499 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import scala.util.Random
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.ml.param.ParamsSuite
+import org.apache.spark.ml.util.MLTestingUtils
+import org.apache.spark.mllib.classification.LogisticRegressionSuite._
+import org.apache.spark.mllib.linalg.{BLAS, DenseVector, Vectors}
+import org.apache.spark.mllib.random._
+import org.apache.spark.mllib.regression.LabeledPoint
+import org.apache.spark.mllib.util.MLlibTestSparkContext
+import org.apache.spark.mllib.util.TestingUtils._
+import org.apache.spark.sql.{DataFrame, Row}
+
+class GeneralizedLinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
+
+ private val seed: Int = 42
+ @transient var datasetGaussianIdentity: DataFrame = _
+ @transient var datasetGaussianLog: DataFrame = _
+ @transient var datasetGaussianInverse: DataFrame = _
+ @transient var datasetBinomial: DataFrame = _
+ @transient var datasetPoissonLog: DataFrame = _
+ @transient var datasetPoissonIdentity: DataFrame = _
+ @transient var datasetPoissonSqrt: DataFrame = _
+ @transient var datasetGammaInverse: DataFrame = _
+ @transient var datasetGammaIdentity: DataFrame = _
+ @transient var datasetGammaLog: DataFrame = _
+
+ override def beforeAll(): Unit = {
+ super.beforeAll()
+
+ import GeneralizedLinearRegressionSuite._
+
+ datasetGaussianIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "identity"), 2))
+
+ datasetGaussianLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "log"), 2))
+
+ datasetGaussianInverse = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "inverse"), 2))
+
+ datasetBinomial = {
+ val nPoints = 10000
+ val coefficients = Array(-0.57997, 0.912083, -0.371077, -0.819866, 2.688191)
+ val xMean = Array(5.843, 3.057, 3.758, 1.199)
+ val xVariance = Array(0.6856, 0.1899, 3.116, 0.581)
+
+ val testData =
+ generateMultinomialLogisticInput(coefficients, xMean, xVariance, true, nPoints, seed)
+
+ sqlContext.createDataFrame(sc.parallelize(testData, 4))
+ }
+
+ datasetPoissonLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "log"), 2))
+
+ datasetPoissonIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "identity"), 2))
+
+ datasetPoissonSqrt = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "sqrt"), 2))
+
+ datasetGammaInverse = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "inverse"), 2))
+
+ datasetGammaIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "identity"), 2))
+
+ datasetGammaLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "log"), 2))
+ }
+
+ test("params") {
+ ParamsSuite.checkParams(new GeneralizedLinearRegression)
+ val model = new GeneralizedLinearRegressionModel("genLinReg", Vectors.dense(0.0), 0.0)
+ ParamsSuite.checkParams(model)
+ }
+
+ test("generalized linear regression: default params") {
+ val glr = new GeneralizedLinearRegression
+ assert(glr.getLabelCol === "label")
+ assert(glr.getFeaturesCol === "features")
+ assert(glr.getPredictionCol === "prediction")
+ assert(glr.getFitIntercept)
+ assert(glr.getTol === 1E-6)
+ assert(glr.getWeightCol === "")
+ assert(glr.getRegParam === 0.0)
+ assert(glr.getSolver == "irls")
+ val model = glr.setFamily("gaussian").setLink("identity")
+ .fit(datasetGaussianIdentity)
+
+ // copied model must have the same parent.
+ MLTestingUtils.checkCopy(model)
+ model.transform(datasetGaussianIdentity)
+ .select("label", "prediction")
+ .collect()
+
+ assert(model.getFeaturesCol === "features")
+ assert(model.getPredictionCol === "prediction")
+ assert(model.intercept !== 0.0)
+ assert(model.hasParent)
+ assert(model.getFamily === "gaussian")
+ assert(model.getLink === "identity")
+ }
+
+ test("generalized linear regression: gaussian family against glm") {
+ /*
+ R code:
+ f1 <- data$V1 ~ data$V2 + data$V3 - 1
+ f2 <- data$V1 ~ data$V2 + data$V3
+
+ data <- read.csv("path", header=FALSE)
+ for (formula in c(f1, f2)) {
+ model <- glm(formula, family="gaussian", data=data)
+ print(as.vector(coef(model)))
+ }
+
+ [1] 2.2960999 0.8087933
+ [1] 2.5002642 2.2000403 0.5999485
+
+ data <- read.csv("path", header=FALSE)
+ model1 <- glm(f1, family=gaussian(link=log), data=data, start=c(0,0))
+ model2 <- glm(f2, family=gaussian(link=log), data=data, start=c(0,0,0))
+ print(as.vector(coef(model1)))
+ print(as.vector(coef(model2)))
+
+ [1] 0.23069326 0.07993778
+ [1] 0.25001858 0.22002452 0.05998789
+
+ data <- read.csv("path", header=FALSE)
+ for (formula in c(f1, f2)) {
+ model <- glm(formula, family=gaussian(link=inverse), data=data)
+ print(as.vector(coef(model)))
+ }
+
+ [1] 2.3010179 0.8198976
+ [1] 2.4108902 2.2130248 0.6086152
+ */
+
+ val expected = Seq(
+ Vectors.dense(0.0, 2.2960999, 0.8087933),
+ Vectors.dense(2.5002642, 2.2000403, 0.5999485),
+ Vectors.dense(0.0, 0.23069326, 0.07993778),
+ Vectors.dense(0.25001858, 0.22002452, 0.05998789),
+ Vectors.dense(0.0, 2.3010179, 0.8198976),
+ Vectors.dense(2.4108902, 2.2130248, 0.6086152))
+
+ import GeneralizedLinearRegression._
+
+ var idx = 0
+ for ((link, dataset) <- Seq(("identity", datasetGaussianIdentity), ("log", datasetGaussianLog),
+ ("inverse", datasetGaussianInverse))) {
+ for (fitIntercept <- Seq(false, true)) {
+ val trainer = new GeneralizedLinearRegression().setFamily("gaussian").setLink(link)
+ .setFitIntercept(fitIntercept)
+ val model = trainer.fit(dataset)
+ val actual = Vectors.dense(model.intercept, model.coefficients(0), model.coefficients(1))
+ assert(actual ~== expected(idx) absTol 1e-4)
+
+ val familyLink = new FamilyAndLink(Gaussian, Link.fromName(link))
+ model.transform(dataset).select("features", "prediction").collect().foreach {
+ case Row(features: DenseVector, prediction1: Double) =>
+ val eta = BLAS.dot(features, model.coefficients) + model.intercept
+ val prediction2 = familyLink.fitted(eta)
+ assert(prediction1 ~== prediction2 relTol 1E-5)
+ }
+
+ idx += 1
+ }
+ }
+ }
+
+ test("generalized linear regression: gaussian family against glmnet") {
+ /*
+ R code:
+ library(glmnet)
+ data <- read.csv("path", header=FALSE)
+ label = data$V1
+ features = as.matrix(data.frame(data$V2, data$V3))
+ for (intercept in c(FALSE, TRUE)) {
+ for (lambda in c(0.0, 0.1, 1.0)) {
+ model <- glmnet(features, label, family="gaussian", intercept=intercept,
+ lambda=lambda, alpha=0, thresh=1E-14)
+ print(as.vector(coef(model)))
+ }
+ }
+
+ [1] 0.0000000 2.2961005 0.8087932
+ [1] 0.0000000 2.2130368 0.8309556
+ [1] 0.0000000 1.7176137 0.9610657
+ [1] 2.5002642 2.2000403 0.5999485
+ [1] 3.1106389 2.0935142 0.5712711
+ [1] 6.7597127 1.4581054 0.3994266
+ */
+
+ val expected = Seq(
+ Vectors.dense(0.0, 2.2961005, 0.8087932),
+ Vectors.dense(0.0, 2.2130368, 0.8309556),
+ Vectors.dense(0.0, 1.7176137, 0.9610657),
+ Vectors.dense(2.5002642, 2.2000403, 0.5999485),
+ Vectors.dense(3.1106389, 2.0935142, 0.5712711),
+ Vectors.dense(6.7597127, 1.4581054, 0.3994266))
+
+ var idx = 0
+ for (fitIntercept <- Seq(false, true);
+ regParam <- Seq(0.0, 0.1, 1.0)) {
+ val trainer = new GeneralizedLinearRegression().setFamily("gaussian")
+ .setFitIntercept(fitIntercept).setRegParam(regParam)
+ val model = trainer.fit(datasetGaussianIdentity)
+ val actual = Vectors.dense(model.intercept, model.coefficients(0), model.coefficients(1))
+ assert(actual ~== expected(idx) absTol 1e-4)
+
+ idx += 1
+ }
+ }
+
+ test("generalized linear regression: binomial family against glm") {
+ /*
+ R code:
+ f1 <- data$V1 ~ data$V2 + data$V3 + data$V4 + data$V5 - 1
+ f2 <- data$V1 ~ data$V2 + data$V3 + data$V4 + data$V5
+ data <- read.csv("path", header=FALSE)
+
+ for (formula in c(f1, f2)) {
+ model <- glm(formula, family="binomial", data=data)
+ print(as.vector(coef(model)))
+ }
+
+ [1] -0.3560284 1.3010002 -0.3570805 -0.7406762
+ [1] 2.8367406 -0.5896187 0.8931655 -0.3925169 -0.7996989
+
+ for (formula in c(f1, f2)) {
+ model <- glm(formula, family=binomial(link=probit), data=data)
+ print(as.vector(coef(model)))
+ }
+
+ [1] -0.2134390 0.7800646 -0.2144267 -0.4438358
+ [1] 1.6995366 -0.3524694 0.5332651 -0.2352985 -0.4780850
+
+ for (formula in c(f1, f2)) {
+ model <- glm(formula, family=binomial(link=cloglog), data=data)
+ print(as.vector(coef(model)))
+ }
+
+ [1] -0.2832198 0.8434144 -0.2524727 -0.5293452
+ [1] 1.5063590 -0.4038015 0.6133664 -0.2687882 -0.5541758
+ */
+ val expected = Seq(
+ Vectors.dense(0.0, -0.3560284, 1.3010002, -0.3570805, -0.7406762),
+ Vectors.dense(2.8367406, -0.5896187, 0.8931655, -0.3925169, -0.7996989),
+ Vectors.dense(0.0, -0.2134390, 0.7800646, -0.2144267, -0.4438358),
+ Vectors.dense(1.6995366, -0.3524694, 0.5332651, -0.2352985, -0.4780850),
+ Vectors.dense(0.0, -0.2832198, 0.8434144, -0.2524727, -0.5293452),
+ Vectors.dense(1.5063590, -0.4038015, 0.6133664, -0.2687882, -0.5541758))
+
+ import GeneralizedLinearRegression._
+
+ var idx = 0
+ for ((link, dataset) <- Seq(("logit", datasetBinomial), ("probit", datasetBinomial),
+ ("cloglog", datasetBinomial))) {
+ for (fitIntercept <- Seq(false, true)) {
+ val trainer = new GeneralizedLinearRegression().setFamily("binomial").setLink(link)
+ .setFitIntercept(fitIntercept)
+ val model = trainer.fit(dataset)
+ val actual = Vectors.dense(model.intercept, model.coefficients(0), model.coefficients(1),
+ model.coefficients(2), model.coefficients(3))
+ assert(actual ~== expected(idx) absTol 1e-4)
+
+ val familyLink = new FamilyAndLink(Binomial, Link.fromName(link))
+ model.transform(dataset).select("features", "prediction").collect().foreach {
+ case Row(features: DenseVector, prediction1: Double) =>
+ val eta = BLAS.dot(features, model.coefficients) + model.intercept
+ val prediction2 = familyLink.fitted(eta)
+ assert(prediction1 ~== prediction2 relTol 1E-5)
+ }
+
+ idx += 1
+ }
+ }
+ }
+
+ test("generalized linear regression: poisson family against glm") {
+ /*
+ R code:
+ f1 <- data$V1 ~ data$V2 + data$V3 - 1
+ f2 <- data$V1 ~ data$V2 + data$V3
+
+ data <- read.csv("path", header=FALSE)
+ for (formula in c(f1, f2)) {
+ model <- glm(formula, family="poisson", data=data)
+ print(as.vector(coef(model)))
+ }
+
+ [1] 0.22999393 0.08047088
+ [1] 0.25022353 0.21998599 0.05998621
+
+ data <- read.csv("path", header=FALSE)
+ for (formula in c(f1, f2)) {
+ model <- glm(formula, family=poisson(link=identity), data=data)
+ print(as.vector(coef(model)))
+ }
+
+ [1] 2.2929501 0.8119415
+ [1] 2.5012730 2.1999407 0.5999107
+
+ data <- read.csv("path", header=FALSE)
+ for (formula in c(f1, f2)) {
+ model <- glm(formula, family=poisson(link=sqrt), data=data)
+ print(as.vector(coef(model)))
+ }
+
+ [1] 2.2958947 0.8090515
+ [1] 2.5000480 2.1999972 0.5999968
+ */
+ val expected = Seq(
+ Vectors.dense(0.0, 0.22999393, 0.08047088),
+ Vectors.dense(0.25022353, 0.21998599, 0.05998621),
+ Vectors.dense(0.0, 2.2929501, 0.8119415),
+ Vectors.dense(2.5012730, 2.1999407, 0.5999107),
+ Vectors.dense(0.0, 2.2958947, 0.8090515),
+ Vectors.dense(2.5000480, 2.1999972, 0.5999968))
+
+ import GeneralizedLinearRegression._
+
+ var idx = 0
+ for ((link, dataset) <- Seq(("log", datasetPoissonLog), ("identity", datasetPoissonIdentity),
+ ("sqrt", datasetPoissonSqrt))) {
+ for (fitIntercept <- Seq(false, true)) {
+ val trainer = new GeneralizedLinearRegression().setFamily("poisson").setLink(link)
+ .setFitIntercept(fitIntercept)
+ val model = trainer.fit(dataset)
+ val actual = Vectors.dense(model.intercept, model.coefficients(0), model.coefficients(1))
+ assert(actual ~== expected(idx) absTol 1e-4)
+
+ val familyLink = new FamilyAndLink(Poisson, Link.fromName(link))
+ model.transform(dataset).select("features", "prediction").collect().foreach {
+ case Row(features: DenseVector, prediction1: Double) =>
+ val eta = BLAS.dot(features, model.coefficients) + model.intercept
+ val prediction2 = familyLink.fitted(eta)
+ assert(prediction1 ~== prediction2 relTol 1E-5)
+ }
+
+ idx += 1
+ }
+ }
+ }
+
+ test("generalized linear regression: gamma family against glm") {
+ /*
+ R code:
+ f1 <- data$V1 ~ data$V2 + data$V3 - 1
+ f2 <- data$V1 ~ data$V2 + data$V3
+
+ data <- read.csv("path", header=FALSE)
+ for (formula in c(f1, f2)) {
+ model <- glm(formula, family="Gamma", data=data)
+ print(as.vector(coef(model)))
+ }
+
+ [1] 2.3392419 0.8058058
+ [1] 2.3507700 2.2533574 0.6042991
+
+ data <- read.csv("path", header=FALSE)
+ for (formula in c(f1, f2)) {
+ model <- glm(formula, family=Gamma(link=identity), data=data)
+ print(as.vector(coef(model)))
+ }
+
+ [1] 2.2908883 0.8147796
+ [1] 2.5002406 2.1998346 0.6000059
+
+ data <- read.csv("path", header=FALSE)
+ for (formula in c(f1, f2)) {
+ model <- glm(formula, family=Gamma(link=log), data=data)
+ print(as.vector(coef(model)))
+ }
+
+ [1] 0.22958970 0.08091066
+ [1] 0.25003210 0.21996957 0.06000215
+ */
+ val expected = Seq(
+ Vectors.dense(0.0, 2.3392419, 0.8058058),
+ Vectors.dense(2.3507700, 2.2533574, 0.6042991),
+ Vectors.dense(0.0, 2.2908883, 0.8147796),
+ Vectors.dense(2.5002406, 2.1998346, 0.6000059),
+ Vectors.dense(0.0, 0.22958970, 0.08091066),
+ Vectors.dense(0.25003210, 0.21996957, 0.06000215))
+
+ import GeneralizedLinearRegression._
+
+ var idx = 0
+ for ((link, dataset) <- Seq(("inverse", datasetGammaInverse),
+ ("identity", datasetGammaIdentity), ("log", datasetGammaLog))) {
+ for (fitIntercept <- Seq(false, true)) {
+ val trainer = new GeneralizedLinearRegression().setFamily("gamma").setLink(link)
+ .setFitIntercept(fitIntercept)
+ val model = trainer.fit(dataset)
+ val actual = Vectors.dense(model.intercept, model.coefficients(0), model.coefficients(1))
+ assert(actual ~== expected(idx) absTol 1e-4)
+
+ val familyLink = new FamilyAndLink(Gamma, Link.fromName(link))
+ model.transform(dataset).select("features", "prediction").collect().foreach {
+ case Row(features: DenseVector, prediction1: Double) =>
+ val eta = BLAS.dot(features, model.coefficients) + model.intercept
+ val prediction2 = familyLink.fitted(eta)
+ assert(prediction1 ~== prediction2 relTol 1E-5)
+ }
+
+ idx += 1
+ }
+ }
+ }
+}
+
+object GeneralizedLinearRegressionSuite {
+
+ def generateGeneralizedLinearRegressionInput(
--- End diff --
This is an internal method, but it is useful to explain at least `eps` in the API doc. Btw, `eps` is not a good arg name because it doesn't tell what it does. How about `noiseLevel`? Then it is not necessary to provide the API doc since all arg names are self-explained.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by sethah <gi...@git.apache.org>.
Github user sethah commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r52403617
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,472 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.Logging
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
--- End diff --
This line causes a `NoSuchElementException` when you call `fit` without setting a `Link` function. IMO if you set a `Family` for the Estimator, and that family has a default `Link`, then you shouldn't have to explicitly set the `Link` function. In the setter for `Family`, can we check `if !isSet(link)` and set link to the default link?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r54508387
--- Diff: mllib/src/test/scala/org/apache/spark/ml/regression/GeneralizedLinearRegressionSuite.scala ---
@@ -0,0 +1,499 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import scala.util.Random
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.ml.param.ParamsSuite
+import org.apache.spark.ml.util.MLTestingUtils
+import org.apache.spark.mllib.classification.LogisticRegressionSuite._
+import org.apache.spark.mllib.linalg.{BLAS, DenseVector, Vectors}
+import org.apache.spark.mllib.random._
+import org.apache.spark.mllib.regression.LabeledPoint
+import org.apache.spark.mllib.util.MLlibTestSparkContext
+import org.apache.spark.mllib.util.TestingUtils._
+import org.apache.spark.sql.{DataFrame, Row}
+
+class GeneralizedLinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
+
+ private val seed: Int = 42
+ @transient var datasetGaussianIdentity: DataFrame = _
+ @transient var datasetGaussianLog: DataFrame = _
+ @transient var datasetGaussianInverse: DataFrame = _
+ @transient var datasetBinomial: DataFrame = _
+ @transient var datasetPoissonLog: DataFrame = _
+ @transient var datasetPoissonIdentity: DataFrame = _
+ @transient var datasetPoissonSqrt: DataFrame = _
+ @transient var datasetGammaInverse: DataFrame = _
+ @transient var datasetGammaIdentity: DataFrame = _
+ @transient var datasetGammaLog: DataFrame = _
+
+ override def beforeAll(): Unit = {
+ super.beforeAll()
+
+ import GeneralizedLinearRegressionSuite._
+
+ datasetGaussianIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "identity"), 2))
+
+ datasetGaussianLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "log"), 2))
+
+ datasetGaussianInverse = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "inverse"), 2))
+
+ datasetBinomial = {
+ val nPoints = 10000
+ val coefficients = Array(-0.57997, 0.912083, -0.371077, -0.819866, 2.688191)
+ val xMean = Array(5.843, 3.057, 3.758, 1.199)
+ val xVariance = Array(0.6856, 0.1899, 3.116, 0.581)
+
+ val testData =
+ generateMultinomialLogisticInput(coefficients, xMean, xVariance, true, nPoints, seed)
+
+ sqlContext.createDataFrame(sc.parallelize(testData, 4))
+ }
+
+ datasetPoissonLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "log"), 2))
+
+ datasetPoissonIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "identity"), 2))
+
+ datasetPoissonSqrt = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "sqrt"), 2))
+
+ datasetGammaInverse = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "inverse"), 2))
+
+ datasetGammaIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "identity"), 2))
+
+ datasetGammaLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "log"), 2))
+ }
+
+ test("params") {
+ ParamsSuite.checkParams(new GeneralizedLinearRegression)
+ val model = new GeneralizedLinearRegressionModel("genLinReg", Vectors.dense(0.0), 0.0)
+ ParamsSuite.checkParams(model)
+ }
+
+ test("generalized linear regression: default params") {
+ val glr = new GeneralizedLinearRegression
+ assert(glr.getLabelCol === "label")
+ assert(glr.getFeaturesCol === "features")
+ assert(glr.getPredictionCol === "prediction")
+ assert(glr.getFitIntercept)
+ assert(glr.getTol === 1E-6)
+ assert(glr.getWeightCol === "")
+ assert(glr.getRegParam === 0.0)
+ assert(glr.getSolver == "irls")
+ val model = glr.setFamily("gaussian").setLink("identity")
+ .fit(datasetGaussianIdentity)
+
+ // copied model must have the same parent.
+ MLTestingUtils.checkCopy(model)
+ model.transform(datasetGaussianIdentity)
+ .select("label", "prediction")
+ .collect()
+
+ assert(model.getFeaturesCol === "features")
+ assert(model.getPredictionCol === "prediction")
+ assert(model.intercept !== 0.0)
+ assert(model.hasParent)
+ assert(model.getFamily === "gaussian")
+ assert(model.getLink === "identity")
+ }
+
+ test("generalized linear regression: gaussian family against glm") {
+ /*
+ R code:
+ f1 <- data$V1 ~ data$V2 + data$V3 - 1
+ f2 <- data$V1 ~ data$V2 + data$V3
+
+ data <- read.csv("path", header=FALSE)
--- End diff --
Shall we provide a test ignored by default that can write the test datasets to disk? Then people can enable it to get the test datasets easily in R. Otherwise, it is not easy for others to verify the result in R.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r54059552
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,565 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionBase extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * Default is "gaussian".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "The name of family which is a description of the error distribution to be used in the " +
+ "model. Supported options: gaussian(default), binomial, poisson and gamma.",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilyNames.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of link function which provides the relationship
+ * between the linear predictor and the mean of the distribution function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "The name of link function " +
+ "which provides the relationship between the linear predictor and the mean of the " +
+ "distribution function. Supported options: identity, log, inverse, logit, probit, " +
+ "cloglog and sqrt.",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinkNames.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ import GeneralizedLinearRegression._
+ protected lazy val familyObj = Family.fromName($(family))
+ protected lazy val linkObj = if (isDefined(link)) {
+ Link.fromName($(link))
+ } else {
+ familyObj.defaultLink
+ }
+ protected lazy val familyAndLink = new FamilyAndLink(familyObj, linkObj)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ if ($(solver) == "irls") {
+ setDefault(maxIter -> 25)
+ }
+ if (isDefined(link)) {
+ require(GeneralizedLinearRegression.supportedFamilyAndLinkParis.contains(
+ familyObj -> linkObj), s"Generalized Linear Regression with ${$(family)} family " +
+ s"does not support ${$(link)} link function.")
+ }
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor (link function) and
+ * a description of the error distribution (family).
+ * It supports "gaussian", "binomial", "poisson" and "gamma" as family.
+ * Valid link functions for each family is listed below. The first link function of each family
+ * is the default one.
+ * - "gaussian" -> "identity", "log", "inverse"
+ * - "binomial" -> "logit", "probit", "cloglog"
+ * - "poisson" -> "log", "identity", "sqrt"
+ * - "gamma" -> "inverse", "identity", "log"
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionBase with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("glm"))
+
+ /**
+ * Sets the value of param [[family]].
+ * Default is "gaussian".
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+ setDefault(family -> "gaussian")
+
+ /**
+ * Sets the value of param [[link]].
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Sets if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+
+ /**
+ * Sets the maximum number of iterations.
+ * Default is 25 if the solver algorithm is "irls".
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
--- End diff --
This is a little tricky to do. I saw your changes on L87. I think it is a little confusing because `explainParams` only shows the current default values instead of explaining the logic. And it happens in `validateParams`, which means `explainParams` won't display `25` as the default value for `irls`.
We can postpone this discussion since we only support IRLS for now.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by AmplabJenkins <gi...@git.apache.org>.
Github user AmplabJenkins commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-182961035
Merged build finished. Test PASSed.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-190529580
**[Test build #52215 has started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/52215/consoleFull)** for PR 11136 at commit [`314b562`](https://github.com/apache/spark/commit/314b562f315723a7117851289c8f5b6e1b16a6ac).
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by AmplabJenkins <gi...@git.apache.org>.
Github user AmplabJenkins commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-188745899
Merged build finished. Test PASSed.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/11136#issuecomment-190537367
**[Test build #52216 has started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/52216/consoleFull)** for PR 11136 at commit [`31a912c`](https://github.com/apache/spark/commit/31a912cd74cf3dffbf8cc0af8c57b777d49579eb).
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r54059489
--- Diff: mllib/src/test/scala/org/apache/spark/ml/regression/GeneralizedLinearRegressionSuite.scala ---
@@ -0,0 +1,499 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import scala.util.Random
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.ml.param.ParamsSuite
+import org.apache.spark.ml.util.MLTestingUtils
+import org.apache.spark.mllib.classification.LogisticRegressionSuite._
+import org.apache.spark.mllib.linalg.{BLAS, DenseVector, Vectors}
+import org.apache.spark.mllib.random._
+import org.apache.spark.mllib.regression.LabeledPoint
+import org.apache.spark.mllib.util.MLlibTestSparkContext
+import org.apache.spark.mllib.util.TestingUtils._
+import org.apache.spark.sql.{DataFrame, Row}
+
+class GeneralizedLinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
+
+ private val seed: Int = 42
+ @transient var datasetGaussianIdentity: DataFrame = _
+ @transient var datasetGaussianLog: DataFrame = _
+ @transient var datasetGaussianInverse: DataFrame = _
+ @transient var datasetBinomial: DataFrame = _
+ @transient var datasetPoissonLog: DataFrame = _
+ @transient var datasetPoissonIdentity: DataFrame = _
+ @transient var datasetPoissonSqrt: DataFrame = _
+ @transient var datasetGammaInverse: DataFrame = _
+ @transient var datasetGammaIdentity: DataFrame = _
+ @transient var datasetGammaLog: DataFrame = _
+
+ override def beforeAll(): Unit = {
+ super.beforeAll()
+
+ import GeneralizedLinearRegressionSuite._
+
+ datasetGaussianIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "identity"), 2))
+
+ datasetGaussianLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "log"), 2))
+
+ datasetGaussianInverse = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "inverse"), 2))
+
+ datasetBinomial = {
+ val nPoints = 10000
+ val coefficients = Array(-0.57997, 0.912083, -0.371077, -0.819866, 2.688191)
+ val xMean = Array(5.843, 3.057, 3.758, 1.199)
+ val xVariance = Array(0.6856, 0.1899, 3.116, 0.581)
+
+ val testData =
+ generateMultinomialLogisticInput(coefficients, xMean, xVariance, true, nPoints, seed)
+
+ sqlContext.createDataFrame(sc.parallelize(testData, 4))
+ }
+
+ datasetPoissonLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "log"), 2))
+
+ datasetPoissonIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "identity"), 2))
+
+ datasetPoissonSqrt = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "sqrt"), 2))
+
+ datasetGammaInverse = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "inverse"), 2))
+
+ datasetGammaIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "identity"), 2))
+
+ datasetGammaLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "log"), 2))
+ }
+
+ test("params") {
+ ParamsSuite.checkParams(new GeneralizedLinearRegression)
+ val model = new GeneralizedLinearRegressionModel("genLinReg", Vectors.dense(0.0), 0.0)
+ ParamsSuite.checkParams(model)
+ }
+
+ test("generalized linear regression: default params") {
+ val glr = new GeneralizedLinearRegression
+ assert(glr.getLabelCol === "label")
+ assert(glr.getFeaturesCol === "features")
+ assert(glr.getPredictionCol === "prediction")
+ assert(glr.getFitIntercept)
+ assert(glr.getTol === 1E-6)
+ assert(glr.getWeightCol === "")
+ assert(glr.getRegParam === 0.0)
+ assert(glr.getSolver == "irls")
+ val model = glr.setFamily("gaussian").setLink("identity")
+ .fit(datasetGaussianIdentity)
+
+ // copied model must have the same parent.
+ MLTestingUtils.checkCopy(model)
+ model.transform(datasetGaussianIdentity)
+ .select("label", "prediction")
+ .collect()
+
+ assert(model.getFeaturesCol === "features")
+ assert(model.getPredictionCol === "prediction")
+ assert(model.intercept !== 0.0)
+ assert(model.hasParent)
+ assert(model.getFamily === "gaussian")
+ assert(model.getLink === "identity")
+ }
+
+ test("generalized linear regression: gaussian family against glm") {
+ /*
+ R code:
+ f1 <- data$V1 ~ data$V2 + data$V3 - 1
+ f2 <- data$V1 ~ data$V2 + data$V3
+
+ data <- read.csv("path", header=FALSE)
+ for (formula in c(f1, f2)) {
+ model <- glm(formula, family="gaussian", data=data)
+ print(as.vector(coef(model)))
+ }
+
+ [1] 2.2960999 0.8087933
+ [1] 2.5002642 2.2000403 0.5999485
+
+ data <- read.csv("path", header=FALSE)
+ model1 <- glm(f1, family=gaussian(link=log), data=data, start=c(0,0))
+ model2 <- glm(f2, family=gaussian(link=log), data=data, start=c(0,0,0))
+ print(as.vector(coef(model1)))
+ print(as.vector(coef(model2)))
+
+ [1] 0.23069326 0.07993778
+ [1] 0.25001858 0.22002452 0.05998789
+
+ data <- read.csv("path", header=FALSE)
+ for (formula in c(f1, f2)) {
+ model <- glm(formula, family=gaussian(link=inverse), data=data)
+ print(as.vector(coef(model)))
+ }
+
+ [1] 2.3010179 0.8198976
+ [1] 2.4108902 2.2130248 0.6086152
+ */
+
+ val expected = Seq(
+ Vectors.dense(0.0, 2.2960999, 0.8087933),
+ Vectors.dense(2.5002642, 2.2000403, 0.5999485),
+ Vectors.dense(0.0, 0.23069326, 0.07993778),
+ Vectors.dense(0.25001858, 0.22002452, 0.05998789),
+ Vectors.dense(0.0, 2.3010179, 0.8198976),
+ Vectors.dense(2.4108902, 2.2130248, 0.6086152))
+
+ import GeneralizedLinearRegression._
+
+ var idx = 0
+ for ((link, dataset) <- Seq(("identity", datasetGaussianIdentity), ("log", datasetGaussianLog),
+ ("inverse", datasetGaussianInverse))) {
+ for (fitIntercept <- Seq(false, true)) {
+ val trainer = new GeneralizedLinearRegression().setFamily("gaussian").setLink(link)
+ .setFitIntercept(fitIntercept)
+ val model = trainer.fit(dataset)
+ val actual = Vectors.dense(model.intercept, model.coefficients(0), model.coefficients(1))
+ assert(actual ~== expected(idx) absTol 1e-4)
+
+ val familyLink = new FamilyAndLink(Gaussian, Link.fromName(link))
+ model.transform(dataset).select("features", "prediction").collect().foreach {
+ case Row(features: DenseVector, prediction1: Double) =>
+ val eta = BLAS.dot(features, model.coefficients) + model.intercept
+ val prediction2 = familyLink.fitted(eta)
+ assert(prediction1 ~== prediction2 relTol 1E-5)
+ }
+
+ idx += 1
+ }
+ }
+ }
+
+ test("generalized linear regression: gaussian family against glmnet") {
--- End diff --
Sounds good.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by sethah <gi...@git.apache.org>.
Github user sethah commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r52400530
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,472 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.Logging
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedLinks.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getLink: String = $(link)
+
+ @Since("2.0.0")
+ override def validateParams(): Unit = {
+ require(GeneralizedLinearRegression.supportedFamilyLinkPairs.contains($(family) -> $(link)),
+ s"Generalized Linear Regression with ${$(family)} family does not support ${$(link)} " +
+ s"link function.")
+ }
+}
+
+/**
+ * :: Experimental ::
+ *
+ * Fit a Generalized Linear Model ([[https://en.wikipedia.org/wiki/Generalized_linear_model]])
+ * specified by giving a symbolic description of the linear predictor and
+ * a description of the error distribution.
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val uid: String)
+ extends Regressor[Vector, GeneralizedLinearRegression, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams with Logging {
+
+ @Since("2.0.0")
+ def this() = this(Identifiable.randomUID("genLinReg"))
+
+ /**
+ * Set the name of family which is a description of the error distribution
+ * to be used in the model.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFamily(value: String): this.type = set(family, value)
+
+ /**
+ * Set the name of the model link function.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setLink(value: String): this.type = set(link, value)
+
+ /**
+ * Set if we should fit the intercept.
+ * Default is true.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
+ setDefault(fitIntercept -> true)
+
+ /**
+ * Set the maximum number of iterations.
+ * Default is 100.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setMaxIter(value: Int): this.type = set(maxIter, value)
+ setDefault(maxIter -> 100)
+
+ /**
+ * Set the convergence tolerance of iterations.
+ * Smaller value will lead to higher accuracy with the cost of more iterations.
+ * Default is 1E-6.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setTol(value: Double): this.type = set(tol, value)
+ setDefault(tol -> 1E-6)
+
+ /**
+ * Set the regularization parameter.
+ * Default is 0.0.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setRegParam(value: Double): this.type = set(regParam, value)
+ setDefault(regParam -> 0.0)
+
+ /**
+ * Whether to over-/under-sample training instances according to the given weights in weightCol.
+ * If empty, all instances are treated equally (weight 1.0).
+ * Default is empty, so all instances have weight one.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setWeightCol(value: String): this.type = set(weightCol, value)
+ setDefault(weightCol -> "")
+
+ /**
+ * Set the solver algorithm used for optimization.
+ * Currently only support "irls" which is also the default solver.
+ * @group setParam
+ */
+ @Since("2.0.0")
+ def setSolver(value: String): this.type = set(solver, value)
+ setDefault(solver -> "irls")
+
+ override protected def train(dataset: DataFrame): GeneralizedLinearRegressionModel = {
+ val familyLink = $(family) match {
+ case "gaussian" => Gaussian($(link))
+ case "binomial" => Binomial($(link))
+ case "poisson" => Poisson($(link))
+ case "gamma" => Gamma($(link))
+ }
+
+ val w = if ($(weightCol).isEmpty) lit(1.0) else col($(weightCol))
+ val instances: RDD[Instance] = dataset.select(
+ col($(labelCol)), w, col($(featuresCol))).map {
+ case Row(label: Double, weight: Double, features: Vector) =>
+ Instance(label, weight, features)
+ }
+
+ if ($(family) == "gaussian" && $(link) == "identity") {
+ val optimizer = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true)
+ val wlsModel = optimizer.fit(instances)
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ wlsModel.coefficients, wlsModel.intercept).setParent(this))
+ return model
+ }
+
+ val newInstances = instances.map { instance =>
+ val mu = familyLink.initialize(instance.label, instance.weight)
+ val eta = familyLink.predict(mu)
+ Instance(eta, instance.weight, instance.features)
+ }
+
+ val initialModel = new WeightedLeastSquares($(fitIntercept), $(regParam),
+ standardizeFeatures = true, standardizeLabel = true).fit(newInstances)
+
+ val reweightFunc: (Instance, WeightedLeastSquaresModel) => (Double, Double) = {
+ (instance: Instance, model: WeightedLeastSquaresModel) => {
+ val eta = model.predict(instance.features)
+ val mu = familyLink.fitted(eta)
+ val z = familyLink.adjusted(instance.label, mu, eta)
+ val w = familyLink.weights(mu) * instance.weight
+ (z, w)
+ }
+ }
+
+ val optimizer = new IterativelyReweightedLeastSquares(initialModel, reweightFunc,
+ $(fitIntercept), $(regParam), $(maxIter), $(tol))
+
+ val irlsModel = optimizer.fit(instances)
+
+ val model = copyValues(new GeneralizedLinearRegressionModel(uid,
+ irlsModel.coefficients, irlsModel.intercept).setParent(this))
+ model
+ }
+
+ @Since("2.0.0")
+ override def copy(extra: ParamMap): GeneralizedLinearRegression = defaultCopy(extra)
+}
+
+@Since("2.0.0")
+object GeneralizedLinearRegression {
+
+ /** Set of families that GeneralizedLinearRegression supports */
+ private[ml] val supportedFamilies = Set("gaussian", "binomial", "poisson", "gamma")
+
+ /** Set of links that GeneralizedLinearRegression supports */
+ private[ml] val supportedLinks = Set("identity", "log", "inverse", "logit", "probit",
+ "cloglog", "sqrt")
+
+ /** Set of family and link pairs that GeneralizedLinearRegression supports */
+ private[ml] val supportedFamilyLinkPairs = Set(
+ "gaussian" -> "identity", "gaussian" -> "log", "gaussian" -> "inverse",
+ "binomial" -> "logit", "binomial" -> "probit", "binomial" -> "cloglog",
+ "poisson" -> "log", "poisson" -> "identity", "poisson" -> "sqrt",
+ "gamma" -> "inverse", "gamma" -> "identity", "gamma" -> "log"
+ )
+}
+
+/**
+ * :: Experimental ::
+ * Model produced by [[GeneralizedLinearRegression]].
+ */
+@Experimental
+@Since("2.0.0")
+class GeneralizedLinearRegressionModel private[ml] (
+ @Since("2.0.0") override val uid: String,
+ @Since("2.0.0") val coefficients: Vector,
+ @Since("2.0.0") val intercept: Double)
+ extends RegressionModel[Vector, GeneralizedLinearRegressionModel]
+ with GeneralizedLinearRegressionParams {
+
+ private lazy val familyLink = $(family) match {
+ case "gaussian" => Gaussian($(link))
+ case "binomial" => Binomial($(link))
+ case "poisson" => Poisson($(link))
+ case "gamma" => Gamma($(link))
+ }
+
+ override protected def predict(features: Vector): Double = {
+ val eta = BLAS.dot(features, coefficients) + intercept
+ familyLink.fitted(eta)
+ }
+
+ @Since("2.0.0")
+ override def copy(extra: ParamMap): GeneralizedLinearRegressionModel = {
+ copyValues(new GeneralizedLinearRegressionModel(uid, coefficients, intercept), extra)
+ .setParent(parent)
+ }
+}
+
+/**
+ * A description of the error distribution and link function to be used in the model.
+ * @param link a link function instance
+ */
+private[ml] abstract class Family(val link: Link) extends Serializable {
+
+ /** Initialize the starting value for mu. */
+ def initialize(y: Double, weight: Double): Double
+
+ /** The variance of mu to its mean. */
+ def variance(mu: Double): Double
+
+ /** Weights for IRLS steps. */
+ def weights(mu: Double): Double = {
+ 1.0 / (math.pow(this.link.deriv(mu), 2.0) * this.variance(mu))
+ }
+
+ /** The adjusted response variable. */
+ def adjusted(y: Double, mu: Double, eta: Double): Double = {
+ eta + (y - mu) * link.deriv(mu)
+ }
+
+ /** Linear predictors based on given mu. */
+ def predict(mu: Double): Double = this.link.link(mu)
+
+ /** Fitted values based on linear predictors eta. */
+ def fitted(eta: Double): Double = this.link.unlink(eta)
+}
+
+/**
+ * Gaussian exponential family distribution.
+ * The default link for the Gaussian family is the identity link.
+ * @param link a link function instance
+ */
+private[ml] class Gaussian(link: Link = Identity) extends Family(link) {
+
+ override def initialize(y: Double, weight: Double): Double = y
+
+ def variance(mu: Double): Double = 1.0
+}
+
+private[ml] object Gaussian {
+
+ def apply(link: String): Gaussian = {
+ link match {
+ case "identity" => new Gaussian(Identity)
+ case "log" => new Gaussian(Log)
+ case "inverse" => new Gaussian(Inverse)
+ }
+ }
+}
+
+/**
+ * Binomial exponential family distribution.
+ * The default link for the Binomial family is the logit link.
+ * @param link a link function instance
+ */
+private[ml] class Binomial(link: Link = Logit) extends Family(link) {
+
+ override def initialize(y: Double, weight: Double): Double = {
+ (weight * y + 0.5) / (weight + 1.0)
+ }
+
+ override def variance(mu: Double): Double = mu * (1 - mu)
+}
+
+private[ml] object Binomial {
+
+ def apply(link: String): Binomial = {
+ link match {
+ case "logit" => new Binomial(Logit)
+ case "probit" => new Binomial(Probit)
+ case "cloglog" => new Binomial(CLogLog)
+ }
+ }
+}
+
+/**
+ * Poisson exponential family distribution.
+ * The default link for the Poisson family is the log link.
+ * @param link a link function instance
+ */
+private[ml] class Poisson(link: Link = Log) extends Family(link) {
+
+ override def initialize(y: Double, weight: Double): Double = y + 0.1
+
+ override def variance(mu: Double): Double = mu
+}
+
+private[ml] object Poisson {
+
+ def apply(link: String): Poisson = {
+ link match {
+ case "log" => new Poisson(Log)
+ case "sqrt" => new Poisson(Sqrt)
+ case "identity" => new Poisson(Identity)
+ }
+ }
+}
+
+/**
+ * Gamma exponential family distribution.
+ * The default link for the Gamma family is the inverse link.
+ * @param link a link function instance
+ */
+private[ml] class Gamma(link: Link = Log) extends Family(link) {
--- End diff --
Typo in the default link. `Log` => `Inverse`
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r54508392
--- Diff: mllib/src/test/scala/org/apache/spark/ml/regression/GeneralizedLinearRegressionSuite.scala ---
@@ -0,0 +1,499 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import scala.util.Random
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.ml.param.ParamsSuite
+import org.apache.spark.ml.util.MLTestingUtils
+import org.apache.spark.mllib.classification.LogisticRegressionSuite._
+import org.apache.spark.mllib.linalg.{BLAS, DenseVector, Vectors}
+import org.apache.spark.mllib.random._
+import org.apache.spark.mllib.regression.LabeledPoint
+import org.apache.spark.mllib.util.MLlibTestSparkContext
+import org.apache.spark.mllib.util.TestingUtils._
+import org.apache.spark.sql.{DataFrame, Row}
+
+class GeneralizedLinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
+
+ private val seed: Int = 42
+ @transient var datasetGaussianIdentity: DataFrame = _
+ @transient var datasetGaussianLog: DataFrame = _
+ @transient var datasetGaussianInverse: DataFrame = _
+ @transient var datasetBinomial: DataFrame = _
+ @transient var datasetPoissonLog: DataFrame = _
+ @transient var datasetPoissonIdentity: DataFrame = _
+ @transient var datasetPoissonSqrt: DataFrame = _
+ @transient var datasetGammaInverse: DataFrame = _
+ @transient var datasetGammaIdentity: DataFrame = _
+ @transient var datasetGammaLog: DataFrame = _
+
+ override def beforeAll(): Unit = {
+ super.beforeAll()
+
+ import GeneralizedLinearRegressionSuite._
+
+ datasetGaussianIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "identity"), 2))
+
+ datasetGaussianLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "log"), 2))
+
+ datasetGaussianInverse = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gaussian", link = "inverse"), 2))
+
+ datasetBinomial = {
+ val nPoints = 10000
+ val coefficients = Array(-0.57997, 0.912083, -0.371077, -0.819866, 2.688191)
+ val xMean = Array(5.843, 3.057, 3.758, 1.199)
+ val xVariance = Array(0.6856, 0.1899, 3.116, 0.581)
+
+ val testData =
+ generateMultinomialLogisticInput(coefficients, xMean, xVariance, true, nPoints, seed)
+
+ sqlContext.createDataFrame(sc.parallelize(testData, 4))
+ }
+
+ datasetPoissonLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "log"), 2))
+
+ datasetPoissonIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "identity"), 2))
+
+ datasetPoissonSqrt = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "poisson", link = "sqrt"), 2))
+
+ datasetGammaInverse = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "inverse"), 2))
+
+ datasetGammaIdentity = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 2.5, coefficients = Array(2.2, 0.6), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "identity"), 2))
+
+ datasetGammaLog = sqlContext.createDataFrame(
+ sc.parallelize(generateGeneralizedLinearRegressionInput(
+ intercept = 0.25, coefficients = Array(0.22, 0.06), xMean = Array(2.9, 10.5),
+ xVariance = Array(0.7, 1.2), nPoints = 10000, seed, eps = 0.01,
+ family = "gamma", link = "log"), 2))
+ }
+
+ test("params") {
+ ParamsSuite.checkParams(new GeneralizedLinearRegression)
+ val model = new GeneralizedLinearRegressionModel("genLinReg", Vectors.dense(0.0), 0.0)
+ ParamsSuite.checkParams(model)
+ }
+
+ test("generalized linear regression: default params") {
+ val glr = new GeneralizedLinearRegression
+ assert(glr.getLabelCol === "label")
+ assert(glr.getFeaturesCol === "features")
+ assert(glr.getPredictionCol === "prediction")
+ assert(glr.getFitIntercept)
+ assert(glr.getTol === 1E-6)
+ assert(glr.getWeightCol === "")
+ assert(glr.getRegParam === 0.0)
+ assert(glr.getSolver == "irls")
+ val model = glr.setFamily("gaussian").setLink("identity")
+ .fit(datasetGaussianIdentity)
+
+ // copied model must have the same parent.
+ MLTestingUtils.checkCopy(model)
+ model.transform(datasetGaussianIdentity)
+ .select("label", "prediction")
+ .collect()
+
+ assert(model.getFeaturesCol === "features")
+ assert(model.getPredictionCol === "prediction")
+ assert(model.intercept !== 0.0)
+ assert(model.hasParent)
+ assert(model.getFamily === "gaussian")
+ assert(model.getLink === "identity")
+ }
+
+ test("generalized linear regression: gaussian family against glm") {
+ /*
+ R code:
+ f1 <- data$V1 ~ data$V2 + data$V3 - 1
+ f2 <- data$V1 ~ data$V2 + data$V3
+
+ data <- read.csv("path", header=FALSE)
+ for (formula in c(f1, f2)) {
+ model <- glm(formula, family="gaussian", data=data)
+ print(as.vector(coef(model)))
+ }
+
+ [1] 2.2960999 0.8087933
+ [1] 2.5002642 2.2000403 0.5999485
+
+ data <- read.csv("path", header=FALSE)
+ model1 <- glm(f1, family=gaussian(link=log), data=data, start=c(0,0))
+ model2 <- glm(f2, family=gaussian(link=log), data=data, start=c(0,0,0))
+ print(as.vector(coef(model1)))
+ print(as.vector(coef(model2)))
+
+ [1] 0.23069326 0.07993778
+ [1] 0.25001858 0.22002452 0.05998789
+
+ data <- read.csv("path", header=FALSE)
+ for (formula in c(f1, f2)) {
+ model <- glm(formula, family=gaussian(link=inverse), data=data)
+ print(as.vector(coef(model)))
+ }
+
+ [1] 2.3010179 0.8198976
+ [1] 2.4108902 2.2130248 0.6086152
+ */
+
+ val expected = Seq(
+ Vectors.dense(0.0, 2.2960999, 0.8087933),
+ Vectors.dense(2.5002642, 2.2000403, 0.5999485),
+ Vectors.dense(0.0, 0.23069326, 0.07993778),
+ Vectors.dense(0.25001858, 0.22002452, 0.05998789),
+ Vectors.dense(0.0, 2.3010179, 0.8198976),
+ Vectors.dense(2.4108902, 2.2130248, 0.6086152))
+
+ import GeneralizedLinearRegression._
+
+ var idx = 0
+ for ((link, dataset) <- Seq(("identity", datasetGaussianIdentity), ("log", datasetGaussianLog),
+ ("inverse", datasetGaussianInverse))) {
+ for (fitIntercept <- Seq(false, true)) {
+ val trainer = new GeneralizedLinearRegression().setFamily("gaussian").setLink(link)
+ .setFitIntercept(fitIntercept)
+ val model = trainer.fit(dataset)
+ val actual = Vectors.dense(model.intercept, model.coefficients(0), model.coefficients(1))
+ assert(actual ~== expected(idx) absTol 1e-4)
+
+ val familyLink = new FamilyAndLink(Gaussian, Link.fromName(link))
+ model.transform(dataset).select("features", "prediction").collect().foreach {
+ case Row(features: DenseVector, prediction1: Double) =>
+ val eta = BLAS.dot(features, model.coefficients) + model.intercept
+ val prediction2 = familyLink.fitted(eta)
+ assert(prediction1 ~== prediction2 relTol 1E-5)
--- End diff --
Embed current `link` and `fitIntercept` in the error message so we know which one fails the test.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r53722393
--- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
@@ -0,0 +1,547 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.regression
+
+import breeze.stats.distributions.{Gaussian => GD}
+
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.PredictorParams
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.optim._
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared._
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{BLAS, Vector}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions._
+
+/**
+ * Params for Generalized Linear Regression.
+ */
+private[regression] trait GeneralizedLinearRegressionParams extends PredictorParams
+ with HasFitIntercept with HasMaxIter with HasTol with HasRegParam with HasWeightCol
+ with HasSolver with Logging {
+
+ /**
+ * Param for the name of family which is a description of the error distribution
+ * to be used in the model.
+ * Supported options: "gaussian", "binomial", "poisson" and "gamma".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val family: Param[String] = new Param(this, "family",
+ "the name of family which is a description of the error distribution to be used in the model",
+ ParamValidators.inArray[String](GeneralizedLinearRegression.supportedFamilies.toArray))
+
+ /** @group getParam */
+ @Since("2.0.0")
+ def getFamily: String = $(family)
+
+ /**
+ * Param for the name of the model link function.
+ * Supported options: "identity", "log", "inverse", "logit", "probit", "cloglog" and "sqrt".
+ * @group param
+ */
+ @Since("2.0.0")
+ final val link: Param[String] = new Param(this, "link", "the name of the model link function",
--- End diff --
* Include supported options and the default behavior in the param doc and the ScalaDoc.
* Mention the list of valid (family, link) combinations somewhere in the public doc.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-12811] [ML] Estimator for Generalized L...
Posted by mengxr <gi...@git.apache.org>.
Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/11136#discussion_r54059524
--- Diff: mllib/src/main/scala/org/apache/spark/ml/optim/WeightedLeastSquares.scala ---
@@ -156,6 +156,8 @@ private[ml] class WeightedLeastSquares(
private[ml] object WeightedLeastSquares {
+ val MaxNumFeatures: Int = 4096
--- End diff --
add doc
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org