You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by yl...@apache.org on 2017/06/30 12:02:21 UTC
spark git commit: [SPARK-18710][ML] Add offset in GLM
Repository: spark
Updated Branches:
refs/heads/master 52981715b -> 49d767d83
[SPARK-18710][ML] Add offset in GLM
## What changes were proposed in this pull request?
Add support for offset in GLM. This is useful for at least two reasons:
1. Account for exposure: e.g., when modeling the number of accidents, we may need to use miles driven as an offset to access factors on frequency.
2. Test incremental effects of new variables: we can use predictions from the existing model as offset and run a much smaller model on only new variables. This avoids re-estimating the large model with all variables (old + new) and can be very important for efficient large-scaled analysis.
## How was this patch tested?
New test.
yanboliang srowen felixcheung sethah
Author: actuaryzhang <ac...@gmail.com>
Closes #16699 from actuaryzhang/offset.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/49d767d8
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/49d767d8
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/49d767d8
Branch: refs/heads/master
Commit: 49d767d838691fc7d964be2c4349662f5500ff2b
Parents: 5298171
Author: actuaryzhang <ac...@gmail.com>
Authored: Fri Jun 30 20:02:15 2017 +0800
Committer: Yanbo Liang <yb...@gmail.com>
Committed: Fri Jun 30 20:02:15 2017 +0800
----------------------------------------------------------------------
.../org/apache/spark/ml/feature/Instance.scala | 21 +
.../IterativelyReweightedLeastSquares.scala | 14 +-
.../spark/ml/optim/WeightedLeastSquares.scala | 2 +-
.../GeneralizedLinearRegression.scala | 184 ++++--
...IterativelyReweightedLeastSquaresSuite.scala | 40 +-
.../GeneralizedLinearRegressionSuite.scala | 634 +++++++++++--------
6 files changed, 534 insertions(+), 361 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/49d767d8/mllib/src/main/scala/org/apache/spark/ml/feature/Instance.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/Instance.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/Instance.scala
index cce3ca4..dd56fbb 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/Instance.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/Instance.scala
@@ -27,3 +27,24 @@ import org.apache.spark.ml.linalg.Vector
* @param features The vector of features for this data point.
*/
private[ml] case class Instance(label: Double, weight: Double, features: Vector)
+
+/**
+ * Case class that represents an instance of data point with
+ * label, weight, offset and features.
+ * This is mainly used in GeneralizedLinearRegression currently.
+ *
+ * @param label Label for this data point.
+ * @param weight The weight of this instance.
+ * @param offset The offset used for this data point.
+ * @param features The vector of features for this data point.
+ */
+private[ml] case class OffsetInstance(
+ label: Double,
+ weight: Double,
+ offset: Double,
+ features: Vector) {
+
+ /** Converts to an [[Instance]] object by leaving out the offset. */
+ def toInstance: Instance = Instance(label, weight, features)
+
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/49d767d8/mllib/src/main/scala/org/apache/spark/ml/optim/IterativelyReweightedLeastSquares.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/optim/IterativelyReweightedLeastSquares.scala b/mllib/src/main/scala/org/apache/spark/ml/optim/IterativelyReweightedLeastSquares.scala
index 9c49551..6961b45 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/optim/IterativelyReweightedLeastSquares.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/optim/IterativelyReweightedLeastSquares.scala
@@ -18,7 +18,7 @@
package org.apache.spark.ml.optim
import org.apache.spark.internal.Logging
-import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.feature.{Instance, OffsetInstance}
import org.apache.spark.ml.linalg._
import org.apache.spark.rdd.RDD
@@ -43,7 +43,7 @@ private[ml] class IterativelyReweightedLeastSquaresModel(
* find M-estimator in robust regression and other optimization problems.
*
* @param initialModel the initial guess model.
- * @param reweightFunc the reweight function which is used to update offsets and weights
+ * @param reweightFunc the reweight function which is used to update working labels and weights
* at each iteration.
* @param fitIntercept whether to fit intercept.
* @param regParam L2 regularization parameter used by WLS.
@@ -57,13 +57,13 @@ private[ml] class IterativelyReweightedLeastSquaresModel(
*/
private[ml] class IterativelyReweightedLeastSquares(
val initialModel: WeightedLeastSquaresModel,
- val reweightFunc: (Instance, WeightedLeastSquaresModel) => (Double, Double),
+ val reweightFunc: (OffsetInstance, WeightedLeastSquaresModel) => (Double, Double),
val fitIntercept: Boolean,
val regParam: Double,
val maxIter: Int,
val tol: Double) extends Logging with Serializable {
- def fit(instances: RDD[Instance]): IterativelyReweightedLeastSquaresModel = {
+ def fit(instances: RDD[OffsetInstance]): IterativelyReweightedLeastSquaresModel = {
var converged = false
var iter = 0
@@ -75,10 +75,10 @@ private[ml] class IterativelyReweightedLeastSquares(
oldModel = model
- // Update offsets and weights using reweightFunc
+ // Update working labels and weights using reweightFunc
val newInstances = instances.map { instance =>
- val (newOffset, newWeight) = reweightFunc(instance, oldModel)
- Instance(newOffset, newWeight, instance.features)
+ val (newLabel, newWeight) = reweightFunc(instance, oldModel)
+ Instance(newLabel, newWeight, instance.features)
}
// Estimate new model
http://git-wip-us.apache.org/repos/asf/spark/blob/49d767d8/mllib/src/main/scala/org/apache/spark/ml/optim/WeightedLeastSquares.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/optim/WeightedLeastSquares.scala b/mllib/src/main/scala/org/apache/spark/ml/optim/WeightedLeastSquares.scala
index 56ab967..32b0af7 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/optim/WeightedLeastSquares.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/optim/WeightedLeastSquares.scala
@@ -18,7 +18,7 @@
package org.apache.spark.ml.optim
import org.apache.spark.internal.Logging
-import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.feature.{Instance, OffsetInstance}
import org.apache.spark.ml.linalg._
import org.apache.spark.rdd.RDD
http://git-wip-us.apache.org/repos/asf/spark/blob/49d767d8/mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala
index bff0d9b..ce3460a 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala
@@ -26,8 +26,8 @@ import org.apache.spark.SparkException
import org.apache.spark.annotation.{Experimental, Since}
import org.apache.spark.internal.Logging
import org.apache.spark.ml.PredictorParams
-import org.apache.spark.ml.feature.Instance
-import org.apache.spark.ml.linalg.{BLAS, Vector}
+import org.apache.spark.ml.feature.{Instance, OffsetInstance}
+import org.apache.spark.ml.linalg.{BLAS, Vector, Vectors}
import org.apache.spark.ml.optim._
import org.apache.spark.ml.param._
import org.apache.spark.ml.param.shared._
@@ -138,6 +138,27 @@ private[regression] trait GeneralizedLinearRegressionBase extends PredictorParam
@Since("2.0.0")
def getLinkPredictionCol: String = $(linkPredictionCol)
+ /**
+ * Param for offset column name. If this is not set or empty, we treat all instance offsets
+ * as 0.0. The feature specified as offset has a constant coefficient of 1.0.
+ * @group param
+ */
+ @Since("2.3.0")
+ final val offsetCol: Param[String] = new Param[String](this, "offsetCol", "The offset " +
+ "column name. If this is not set or empty, we treat all instance offsets as 0.0")
+
+ /** @group getParam */
+ @Since("2.3.0")
+ def getOffsetCol: String = $(offsetCol)
+
+ /** Checks whether weight column is set and nonempty. */
+ private[regression] def hasWeightCol: Boolean =
+ isSet(weightCol) && $(weightCol).nonEmpty
+
+ /** Checks whether offset column is set and nonempty. */
+ private[regression] def hasOffsetCol: Boolean =
+ isSet(offsetCol) && $(offsetCol).nonEmpty
+
/** Checks whether we should output link prediction. */
private[regression] def hasLinkPredictionCol: Boolean = {
isDefined(linkPredictionCol) && $(linkPredictionCol).nonEmpty
@@ -172,6 +193,11 @@ private[regression] trait GeneralizedLinearRegressionBase extends PredictorParam
}
val newSchema = super.validateAndTransformSchema(schema, fitting, featuresDataType)
+
+ if (hasOffsetCol) {
+ SchemaUtils.checkNumericType(schema, $(offsetCol))
+ }
+
if (hasLinkPredictionCol) {
SchemaUtils.appendColumn(newSchema, $(linkPredictionCol), DoubleType)
} else {
@@ -307,6 +333,16 @@ class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val
def setWeightCol(value: String): this.type = set(weightCol, value)
/**
+ * Sets the value of param [[offsetCol]].
+ * If this is not set or empty, we treat all instance offsets as 0.0.
+ * Default is not set, so all instances have offset 0.0.
+ *
+ * @group setParam
+ */
+ @Since("2.3.0")
+ def setOffsetCol(value: String): this.type = set(offsetCol, value)
+
+ /**
* Sets the solver algorithm used for optimization.
* Currently only supports "irls" which is also the default solver.
*
@@ -329,7 +365,7 @@ class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val
val numFeatures = dataset.select(col($(featuresCol))).first().getAs[Vector](0).size
val instr = Instrumentation.create(this, dataset)
- instr.logParams(labelCol, featuresCol, weightCol, predictionCol, linkPredictionCol,
+ instr.logParams(labelCol, featuresCol, weightCol, offsetCol, predictionCol, linkPredictionCol,
family, solver, fitIntercept, link, maxIter, regParam, tol)
instr.logNumFeatures(numFeatures)
@@ -343,15 +379,16 @@ class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val
"GeneralizedLinearRegression was given data with 0 features, and with Param fitIntercept " +
"set to false. To fit a model with 0 features, fitIntercept must be set to true." )
- val w = if (!isDefined(weightCol) || $(weightCol).isEmpty) lit(1.0) else col($(weightCol))
- val instances: RDD[Instance] =
- dataset.select(col($(labelCol)), w, col($(featuresCol))).rdd.map {
- case Row(label: Double, weight: Double, features: Vector) =>
- Instance(label, weight, features)
- }
+ val w = if (!hasWeightCol) lit(1.0) else col($(weightCol))
+ val offset = if (!hasOffsetCol) lit(0.0) else col($(offsetCol)).cast(DoubleType)
val model = if (familyAndLink.family == Gaussian && familyAndLink.link == Identity) {
// TODO: Make standardizeFeatures and standardizeLabel configurable.
+ val instances: RDD[Instance] =
+ dataset.select(col($(labelCol)), w, offset, col($(featuresCol))).rdd.map {
+ case Row(label: Double, weight: Double, offset: Double, features: Vector) =>
+ Instance(label - offset, weight, features)
+ }
val optimizer = new WeightedLeastSquares($(fitIntercept), $(regParam), elasticNetParam = 0.0,
standardizeFeatures = true, standardizeLabel = true)
val wlsModel = optimizer.fit(instances)
@@ -362,6 +399,11 @@ class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val
wlsModel.diagInvAtWA.toArray, 1, getSolver)
model.setSummary(Some(trainingSummary))
} else {
+ val instances: RDD[OffsetInstance] =
+ dataset.select(col($(labelCol)), w, offset, col($(featuresCol))).rdd.map {
+ case Row(label: Double, weight: Double, offset: Double, features: Vector) =>
+ OffsetInstance(label, weight, offset, features)
+ }
// Fit Generalized Linear Model by iteratively reweighted least squares (IRLS).
val initialModel = familyAndLink.initialize(instances, $(fitIntercept), $(regParam))
val optimizer = new IterativelyReweightedLeastSquares(initialModel,
@@ -425,12 +467,12 @@ object GeneralizedLinearRegression extends DefaultParamsReadable[GeneralizedLine
* Get the initial guess model for [[IterativelyReweightedLeastSquares]].
*/
def initialize(
- instances: RDD[Instance],
+ instances: RDD[OffsetInstance],
fitIntercept: Boolean,
regParam: Double): WeightedLeastSquaresModel = {
val newInstances = instances.map { instance =>
val mu = family.initialize(instance.label, instance.weight)
- val eta = predict(mu)
+ val eta = predict(mu) - instance.offset
Instance(eta, instance.weight, instance.features)
}
// TODO: Make standardizeFeatures and standardizeLabel configurable.
@@ -441,16 +483,16 @@ object GeneralizedLinearRegression extends DefaultParamsReadable[GeneralizedLine
}
/**
- * The reweight function used to update offsets and weights
+ * The reweight function used to update working labels and weights
* at each iteration of [[IterativelyReweightedLeastSquares]].
*/
- val reweightFunc: (Instance, WeightedLeastSquaresModel) => (Double, Double) = {
- (instance: Instance, model: WeightedLeastSquaresModel) => {
- val eta = model.predict(instance.features)
+ val reweightFunc: (OffsetInstance, WeightedLeastSquaresModel) => (Double, Double) = {
+ (instance: OffsetInstance, model: WeightedLeastSquaresModel) => {
+ val eta = model.predict(instance.features) + instance.offset
val mu = fitted(eta)
- val offset = eta + (instance.label - mu) * link.deriv(mu)
- val weight = instance.weight / (math.pow(this.link.deriv(mu), 2.0) * family.variance(mu))
- (offset, weight)
+ val newLabel = eta - instance.offset + (instance.label - mu) * link.deriv(mu)
+ val newWeight = instance.weight / (math.pow(this.link.deriv(mu), 2.0) * family.variance(mu))
+ (newLabel, newWeight)
}
}
}
@@ -950,15 +992,22 @@ class GeneralizedLinearRegressionModel private[ml] (
private lazy val familyAndLink = FamilyAndLink(this)
override protected def predict(features: Vector): Double = {
- val eta = predictLink(features)
+ predict(features, 0.0)
+ }
+
+ /**
+ * Calculates the predicted value when offset is set.
+ */
+ private def predict(features: Vector, offset: Double): Double = {
+ val eta = predictLink(features, offset)
familyAndLink.fitted(eta)
}
/**
- * Calculate the link prediction (linear predictor) of the given instance.
+ * Calculates the link prediction (linear predictor) of the given instance.
*/
- private def predictLink(features: Vector): Double = {
- BLAS.dot(features, coefficients) + intercept
+ private def predictLink(features: Vector, offset: Double): Double = {
+ BLAS.dot(features, coefficients) + intercept + offset
}
override def transform(dataset: Dataset[_]): DataFrame = {
@@ -967,14 +1016,16 @@ class GeneralizedLinearRegressionModel private[ml] (
}
override protected def transformImpl(dataset: Dataset[_]): DataFrame = {
- val predictUDF = udf { (features: Vector) => predict(features) }
- val predictLinkUDF = udf { (features: Vector) => predictLink(features) }
+ val predictUDF = udf { (features: Vector, offset: Double) => predict(features, offset) }
+ val predictLinkUDF = udf { (features: Vector, offset: Double) => predictLink(features, offset) }
+
+ val offset = if (!hasOffsetCol) lit(0.0) else col($(offsetCol)).cast(DoubleType)
var output = dataset
if ($(predictionCol).nonEmpty) {
- output = output.withColumn($(predictionCol), predictUDF(col($(featuresCol))))
+ output = output.withColumn($(predictionCol), predictUDF(col($(featuresCol)), offset))
}
if (hasLinkPredictionCol) {
- output = output.withColumn($(linkPredictionCol), predictLinkUDF(col($(featuresCol))))
+ output = output.withColumn($(linkPredictionCol), predictLinkUDF(col($(featuresCol)), offset))
}
output.toDF()
}
@@ -1146,9 +1197,7 @@ class GeneralizedLinearRegressionSummary private[regression] (
/** Degrees of freedom. */
@Since("2.0.0")
- lazy val degreesOfFreedom: Long = {
- numInstances - rank
- }
+ lazy val degreesOfFreedom: Long = numInstances - rank
/** The residual degrees of freedom. */
@Since("2.0.0")
@@ -1156,18 +1205,20 @@ class GeneralizedLinearRegressionSummary private[regression] (
/** The residual degrees of freedom for the null model. */
@Since("2.0.0")
- lazy val residualDegreeOfFreedomNull: Long = if (model.getFitIntercept) {
- numInstances - 1
- } else {
- numInstances
+ lazy val residualDegreeOfFreedomNull: Long = {
+ if (model.getFitIntercept) numInstances - 1 else numInstances
}
- private def weightCol: Column = {
- if (!model.isDefined(model.weightCol) || model.getWeightCol.isEmpty) {
- lit(1.0)
- } else {
- col(model.getWeightCol)
- }
+ private def label: Column = col(model.getLabelCol).cast(DoubleType)
+
+ private def prediction: Column = col(predictionCol)
+
+ private def weight: Column = {
+ if (!model.hasWeightCol) lit(1.0) else col(model.getWeightCol)
+ }
+
+ private def offset: Column = {
+ if (!model.hasOffsetCol) lit(0.0) else col(model.getOffsetCol).cast(DoubleType)
}
private[regression] lazy val devianceResiduals: DataFrame = {
@@ -1175,25 +1226,23 @@ class GeneralizedLinearRegressionSummary private[regression] (
val r = math.sqrt(math.max(family.deviance(y, mu, weight), 0.0))
if (y > mu) r else -1.0 * r
}
- val w = weightCol
predictions.select(
- drUDF(col(model.getLabelCol), col(predictionCol), w).as("devianceResiduals"))
+ drUDF(label, prediction, weight).as("devianceResiduals"))
}
private[regression] lazy val pearsonResiduals: DataFrame = {
val prUDF = udf { mu: Double => family.variance(mu) }
- val w = weightCol
- predictions.select(col(model.getLabelCol).minus(col(predictionCol))
- .multiply(sqrt(w)).divide(sqrt(prUDF(col(predictionCol)))).as("pearsonResiduals"))
+ predictions.select(label.minus(prediction)
+ .multiply(sqrt(weight)).divide(sqrt(prUDF(prediction))).as("pearsonResiduals"))
}
private[regression] lazy val workingResiduals: DataFrame = {
val wrUDF = udf { (y: Double, mu: Double) => (y - mu) * link.deriv(mu) }
- predictions.select(wrUDF(col(model.getLabelCol), col(predictionCol)).as("workingResiduals"))
+ predictions.select(wrUDF(label, prediction).as("workingResiduals"))
}
private[regression] lazy val responseResiduals: DataFrame = {
- predictions.select(col(model.getLabelCol).minus(col(predictionCol)).as("responseResiduals"))
+ predictions.select(label.minus(prediction).as("responseResiduals"))
}
/**
@@ -1225,16 +1274,35 @@ class GeneralizedLinearRegressionSummary private[regression] (
*/
@Since("2.0.0")
lazy val nullDeviance: Double = {
- val w = weightCol
- val wtdmu: Double = if (model.getFitIntercept) {
- val agg = predictions.agg(sum(w.multiply(col(model.getLabelCol))), sum(w)).first()
- agg.getDouble(0) / agg.getDouble(1)
+ val intercept: Double = if (!model.getFitIntercept) {
+ 0.0
} else {
- link.unlink(0.0)
+ /*
+ Estimate intercept analytically when there is no offset, or when there is offset but
+ the model is Gaussian family with identity link. Otherwise, fit an intercept only model.
+ */
+ if (!model.hasOffsetCol ||
+ (model.hasOffsetCol && family == Gaussian && link == Identity)) {
+ val agg = predictions.agg(sum(weight.multiply(
+ label.minus(offset))), sum(weight)).first()
+ link.link(agg.getDouble(0) / agg.getDouble(1))
+ } else {
+ // Create empty feature column and fit intercept only model using param setting from model
+ val featureNull = "feature_" + java.util.UUID.randomUUID.toString
+ val paramMap = model.extractParamMap()
+ paramMap.put(model.featuresCol, featureNull)
+ if (family.name != "tweedie") {
+ paramMap.remove(model.variancePower)
+ }
+ val emptyVectorUDF = udf{ () => Vectors.zeros(0) }
+ model.parent.fit(
+ dataset.withColumn(featureNull, emptyVectorUDF()), paramMap
+ ).intercept
+ }
}
- predictions.select(col(model.getLabelCol).cast(DoubleType), w).rdd.map {
- case Row(y: Double, weight: Double) =>
- family.deviance(y, wtdmu, weight)
+ predictions.select(label, offset, weight).rdd.map {
+ case Row(y: Double, offset: Double, weight: Double) =>
+ family.deviance(y, link.unlink(intercept + offset), weight)
}.sum()
}
@@ -1243,8 +1311,7 @@ class GeneralizedLinearRegressionSummary private[regression] (
*/
@Since("2.0.0")
lazy val deviance: Double = {
- val w = weightCol
- predictions.select(col(model.getLabelCol).cast(DoubleType), col(predictionCol), w).rdd.map {
+ predictions.select(label, prediction, weight).rdd.map {
case Row(label: Double, pred: Double, weight: Double) =>
family.deviance(label, pred, weight)
}.sum()
@@ -1269,10 +1336,9 @@ class GeneralizedLinearRegressionSummary private[regression] (
/** Akaike Information Criterion (AIC) for the fitted model. */
@Since("2.0.0")
lazy val aic: Double = {
- val w = weightCol
- val weightSum = predictions.select(w).agg(sum(w)).first().getDouble(0)
+ val weightSum = predictions.select(weight).agg(sum(weight)).first().getDouble(0)
val t = predictions.select(
- col(model.getLabelCol).cast(DoubleType), col(predictionCol), w).rdd.map {
+ label, prediction, weight).rdd.map {
case Row(label: Double, pred: Double, weight: Double) =>
(label, pred, weight)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/49d767d8/mllib/src/test/scala/org/apache/spark/ml/optim/IterativelyReweightedLeastSquaresSuite.scala
----------------------------------------------------------------------
diff --git a/mllib/src/test/scala/org/apache/spark/ml/optim/IterativelyReweightedLeastSquaresSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/optim/IterativelyReweightedLeastSquaresSuite.scala
index 5026095..6d14350 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/optim/IterativelyReweightedLeastSquaresSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/optim/IterativelyReweightedLeastSquaresSuite.scala
@@ -18,7 +18,7 @@
package org.apache.spark.ml.optim
import org.apache.spark.SparkFunSuite
-import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.feature.{Instance, OffsetInstance}
import org.apache.spark.ml.linalg.Vectors
import org.apache.spark.ml.util.TestingUtils._
import org.apache.spark.mllib.util.MLlibTestSparkContext
@@ -26,8 +26,8 @@ import org.apache.spark.rdd.RDD
class IterativelyReweightedLeastSquaresSuite extends SparkFunSuite with MLlibTestSparkContext {
- private var instances1: RDD[Instance] = _
- private var instances2: RDD[Instance] = _
+ private var instances1: RDD[OffsetInstance] = _
+ private var instances2: RDD[OffsetInstance] = _
override def beforeAll(): Unit = {
super.beforeAll()
@@ -39,10 +39,10 @@ class IterativelyReweightedLeastSquaresSuite extends SparkFunSuite with MLlibTes
w <- c(1, 2, 3, 4)
*/
instances1 = sc.parallelize(Seq(
- Instance(1.0, 1.0, Vectors.dense(0.0, 5.0).toSparse),
- Instance(0.0, 2.0, Vectors.dense(1.0, 2.0)),
- Instance(1.0, 3.0, Vectors.dense(2.0, 1.0)),
- Instance(0.0, 4.0, Vectors.dense(3.0, 3.0))
+ OffsetInstance(1.0, 1.0, 0.0, Vectors.dense(0.0, 5.0).toSparse),
+ OffsetInstance(0.0, 2.0, 0.0, Vectors.dense(1.0, 2.0)),
+ OffsetInstance(1.0, 3.0, 0.0, Vectors.dense(2.0, 1.0)),
+ OffsetInstance(0.0, 4.0, 0.0, Vectors.dense(3.0, 3.0))
), 2)
/*
R code:
@@ -52,10 +52,10 @@ class IterativelyReweightedLeastSquaresSuite extends SparkFunSuite with MLlibTes
w <- c(1, 2, 3, 4)
*/
instances2 = sc.parallelize(Seq(
- Instance(2.0, 1.0, Vectors.dense(0.0, 5.0).toSparse),
- Instance(8.0, 2.0, Vectors.dense(1.0, 7.0)),
- Instance(3.0, 3.0, Vectors.dense(2.0, 11.0)),
- Instance(9.0, 4.0, Vectors.dense(3.0, 13.0))
+ OffsetInstance(2.0, 1.0, 0.0, Vectors.dense(0.0, 5.0).toSparse),
+ OffsetInstance(8.0, 2.0, 0.0, Vectors.dense(1.0, 7.0)),
+ OffsetInstance(3.0, 3.0, 0.0, Vectors.dense(2.0, 11.0)),
+ OffsetInstance(9.0, 4.0, 0.0, Vectors.dense(3.0, 13.0))
), 2)
}
@@ -156,7 +156,7 @@ class IterativelyReweightedLeastSquaresSuite extends SparkFunSuite with MLlibTes
var idx = 0
for (fitIntercept <- Seq(false, true)) {
val initial = new WeightedLeastSquares(fitIntercept, regParam = 0.0, elasticNetParam = 0.0,
- standardizeFeatures = false, standardizeLabel = false).fit(instances2)
+ standardizeFeatures = false, standardizeLabel = false).fit(instances2.map(_.toInstance))
val irls = new IterativelyReweightedLeastSquares(initial, L1RegressionReweightFunc,
fitIntercept, regParam = 0.0, maxIter = 200, tol = 1e-7).fit(instances2)
val actual = Vectors.dense(irls.intercept, irls.coefficients(0), irls.coefficients(1))
@@ -169,29 +169,29 @@ class IterativelyReweightedLeastSquaresSuite extends SparkFunSuite with MLlibTes
object IterativelyReweightedLeastSquaresSuite {
def BinomialReweightFunc(
- instance: Instance,
+ instance: OffsetInstance,
model: WeightedLeastSquaresModel): (Double, Double) = {
- val eta = model.predict(instance.features)
+ val eta = model.predict(instance.features) + instance.offset
val mu = 1.0 / (1.0 + math.exp(-1.0 * eta))
- val z = eta + (instance.label - mu) / (mu * (1.0 - mu))
+ val z = eta - instance.offset + (instance.label - mu) / (mu * (1.0 - mu))
val w = mu * (1 - mu) * instance.weight
(z, w)
}
def PoissonReweightFunc(
- instance: Instance,
+ instance: OffsetInstance,
model: WeightedLeastSquaresModel): (Double, Double) = {
- val eta = model.predict(instance.features)
+ val eta = model.predict(instance.features) + instance.offset
val mu = math.exp(eta)
- val z = eta + (instance.label - mu) / mu
+ val z = eta - instance.offset + (instance.label - mu) / mu
val w = mu * instance.weight
(z, w)
}
def L1RegressionReweightFunc(
- instance: Instance,
+ instance: OffsetInstance,
model: WeightedLeastSquaresModel): (Double, Double) = {
- val eta = model.predict(instance.features)
+ val eta = model.predict(instance.features) + instance.offset
val e = math.max(math.abs(eta - instance.label), 1e-7)
val w = 1 / e
val y = instance.label
http://git-wip-us.apache.org/repos/asf/spark/blob/49d767d8/mllib/src/test/scala/org/apache/spark/ml/regression/GeneralizedLinearRegressionSuite.scala
----------------------------------------------------------------------
diff --git a/mllib/src/test/scala/org/apache/spark/ml/regression/GeneralizedLinearRegressionSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/regression/GeneralizedLinearRegressionSuite.scala
index f7c7c00..cfaa573 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/regression/GeneralizedLinearRegressionSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/regression/GeneralizedLinearRegressionSuite.scala
@@ -21,7 +21,7 @@ import scala.util.Random
import org.apache.spark.SparkFunSuite
import org.apache.spark.ml.classification.LogisticRegressionSuite._
-import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.feature.{Instance, OffsetInstance}
import org.apache.spark.ml.feature.LabeledPoint
import org.apache.spark.ml.linalg.{BLAS, DenseVector, Vector, Vectors}
import org.apache.spark.ml.param.{ParamMap, ParamsSuite}
@@ -797,77 +797,160 @@ class GeneralizedLinearRegressionSuite
}
}
- test("glm summary: gaussian family with weight") {
+ test("generalized linear regression with weight and offset") {
/*
- R code:
+ R code:
+ library(statmod)
- A <- matrix(c(0, 1, 2, 3, 5, 7, 11, 13), 4, 2)
- b <- c(17, 19, 23, 29)
- w <- c(1, 2, 3, 4)
- df <- as.data.frame(cbind(A, b))
- */
- val datasetWithWeight = Seq(
- Instance(17.0, 1.0, Vectors.dense(0.0, 5.0).toSparse),
- Instance(19.0, 2.0, Vectors.dense(1.0, 7.0)),
- Instance(23.0, 3.0, Vectors.dense(2.0, 11.0)),
- Instance(29.0, 4.0, Vectors.dense(3.0, 13.0))
+ df <- as.data.frame(matrix(c(
+ 0.2, 1.0, 2.0, 0.0, 5.0,
+ 0.5, 2.1, 0.5, 1.0, 2.0,
+ 0.9, 0.4, 1.0, 2.0, 1.0,
+ 0.7, 0.7, 0.0, 3.0, 3.0), 4, 5, byrow = TRUE))
+ families <- list(gaussian, binomial, poisson, Gamma, tweedie(1.5))
+ f1 <- V1 ~ -1 + V4 + V5
+ f2 <- V1 ~ V4 + V5
+ for (f in c(f1, f2)) {
+ for (fam in families) {
+ model <- glm(f, df, family = fam, weights = V2, offset = V3)
+ print(as.vector(coef(model)))
+ }
+ }
+ [1] 0.5169222 -0.3344444
+ [1] 0.9419107 -0.6864404
+ [1] 0.1812436 -0.6568422
+ [1] -0.2869094 0.7857710
+ [1] 0.1055254 0.2979113
+ [1] -0.05990345 0.53188982 -0.32118415
+ [1] -0.2147117 0.9911750 -0.6356096
+ [1] -1.5616130 0.6646470 -0.3192581
+ [1] 0.3390397 -0.3406099 0.6870259
+ [1] 0.3665034 0.1039416 0.1484616
+ */
+ val dataset = Seq(
+ OffsetInstance(0.2, 1.0, 2.0, Vectors.dense(0.0, 5.0)),
+ OffsetInstance(0.5, 2.1, 0.5, Vectors.dense(1.0, 2.0)),
+ OffsetInstance(0.9, 0.4, 1.0, Vectors.dense(2.0, 1.0)),
+ OffsetInstance(0.7, 0.7, 0.0, Vectors.dense(3.0, 3.0))
).toDF()
+
+ val expected = Seq(
+ Vectors.dense(0, 0.5169222, -0.3344444),
+ Vectors.dense(0, 0.9419107, -0.6864404),
+ Vectors.dense(0, 0.1812436, -0.6568422),
+ Vectors.dense(0, -0.2869094, 0.785771),
+ Vectors.dense(0, 0.1055254, 0.2979113),
+ Vectors.dense(-0.05990345, 0.53188982, -0.32118415),
+ Vectors.dense(-0.2147117, 0.991175, -0.6356096),
+ Vectors.dense(-1.561613, 0.664647, -0.3192581),
+ Vectors.dense(0.3390397, -0.3406099, 0.6870259),
+ Vectors.dense(0.3665034, 0.1039416, 0.1484616))
+
+ import GeneralizedLinearRegression._
+
+ var idx = 0
+
+ for (fitIntercept <- Seq(false, true)) {
+ for (family <- Seq("gaussian", "binomial", "poisson", "gamma", "tweedie")) {
+ val trainer = new GeneralizedLinearRegression().setFamily(family)
+ .setFitIntercept(fitIntercept).setOffsetCol("offset")
+ .setWeightCol("weight").setLinkPredictionCol("linkPrediction")
+ if (family == "tweedie") trainer.setVariancePower(1.5)
+ val model = trainer.fit(dataset)
+ val actual = Vectors.dense(model.intercept, model.coefficients(0), model.coefficients(1))
+ assert(actual ~= expected(idx) absTol 1e-4, s"Model mismatch: GLM with family = $family," +
+ s" and fitIntercept = $fitIntercept.")
+
+ val familyLink = FamilyAndLink(trainer)
+ model.transform(dataset).select("features", "offset", "prediction", "linkPrediction")
+ .collect().foreach {
+ case Row(features: DenseVector, offset: Double, prediction1: Double,
+ linkPrediction1: Double) =>
+ val eta = BLAS.dot(features, model.coefficients) + model.intercept + offset
+ val prediction2 = familyLink.fitted(eta)
+ val linkPrediction2 = eta
+ assert(prediction1 ~= prediction2 relTol 1E-5, "Prediction mismatch: GLM with " +
+ s"family = $family, and fitIntercept = $fitIntercept.")
+ assert(linkPrediction1 ~= linkPrediction2 relTol 1E-5, "Link Prediction mismatch: " +
+ s"GLM with family = $family, and fitIntercept = $fitIntercept.")
+ }
+
+ idx += 1
+ }
+ }
+ }
+
+ test("glm summary: gaussian family with weight and offset") {
/*
- R code:
+ R code:
- model <- glm(formula = "b ~ .", family="gaussian", data = df, weights = w)
- summary(model)
+ A <- matrix(c(0, 1, 2, 3, 5, 7, 11, 13), 4, 2)
+ b <- c(17, 19, 23, 29)
+ w <- c(1, 2, 3, 4)
+ off <- c(2, 3, 1, 4)
+ df <- as.data.frame(cbind(A, b))
+ */
+ val dataset = Seq(
+ OffsetInstance(17.0, 1.0, 2.0, Vectors.dense(0.0, 5.0).toSparse),
+ OffsetInstance(19.0, 2.0, 3.0, Vectors.dense(1.0, 7.0)),
+ OffsetInstance(23.0, 3.0, 1.0, Vectors.dense(2.0, 11.0)),
+ OffsetInstance(29.0, 4.0, 4.0, Vectors.dense(3.0, 13.0))
+ ).toDF()
+ /*
+ R code:
- Deviance Residuals:
- 1 2 3 4
- 1.920 -1.358 -1.109 0.960
+ model <- glm(formula = "b ~ .", family = "gaussian", data = df,
+ weights = w, offset = off)
+ summary(model)
- Coefficients:
- Estimate Std. Error t value Pr(>|t|)
- (Intercept) 18.080 9.608 1.882 0.311
- V1 6.080 5.556 1.094 0.471
- V2 -0.600 1.960 -0.306 0.811
+ Deviance Residuals:
+ 1 2 3 4
+ 0.9600 -0.6788 -0.5543 0.4800
- (Dispersion parameter for gaussian family taken to be 7.68)
+ Coefficients:
+ Estimate Std. Error t value Pr(>|t|)
+ (Intercept) 5.5400 4.8040 1.153 0.455
+ V1 -0.9600 2.7782 -0.346 0.788
+ V2 1.7000 0.9798 1.735 0.333
- Null deviance: 202.00 on 3 degrees of freedom
- Residual deviance: 7.68 on 1 degrees of freedom
- AIC: 18.783
+ (Dispersion parameter for gaussian family taken to be 1.92)
- Number of Fisher Scoring iterations: 2
+ Null deviance: 152.10 on 3 degrees of freedom
+ Residual deviance: 1.92 on 1 degrees of freedom
+ AIC: 13.238
- residuals(model, type="pearson")
- 1 2 3 4
- 1.920000 -1.357645 -1.108513 0.960000
+ Number of Fisher Scoring iterations: 2
- residuals(model, type="working")
+ residuals(model, type = "pearson")
+ 1 2 3 4
+ 0.9600000 -0.6788225 -0.5542563 0.4800000
+ residuals(model, type = "working")
1 2 3 4
- 1.92 -0.96 -0.64 0.48
-
- residuals(model, type="response")
+ 0.96 -0.48 -0.32 0.24
+ residuals(model, type = "response")
1 2 3 4
- 1.92 -0.96 -0.64 0.48
+ 0.96 -0.48 -0.32 0.24
*/
val trainer = new GeneralizedLinearRegression()
- .setWeightCol("weight")
+ .setWeightCol("weight").setOffsetCol("offset")
+
+ val model = trainer.fit(dataset)
- val model = trainer.fit(datasetWithWeight)
-
- val coefficientsR = Vectors.dense(Array(6.080, -0.600))
- val interceptR = 18.080
- val devianceResidualsR = Array(1.920, -1.358, -1.109, 0.960)
- val pearsonResidualsR = Array(1.920000, -1.357645, -1.108513, 0.960000)
- val workingResidualsR = Array(1.92, -0.96, -0.64, 0.48)
- val responseResidualsR = Array(1.92, -0.96, -0.64, 0.48)
- val seCoefR = Array(5.556, 1.960, 9.608)
- val tValsR = Array(1.094, -0.306, 1.882)
- val pValsR = Array(0.471, 0.811, 0.311)
- val dispersionR = 7.68
- val nullDevianceR = 202.00
- val residualDevianceR = 7.68
+ val coefficientsR = Vectors.dense(Array(-0.96, 1.7))
+ val interceptR = 5.54
+ val devianceResidualsR = Array(0.96, -0.67882, -0.55426, 0.48)
+ val pearsonResidualsR = Array(0.96, -0.67882, -0.55426, 0.48)
+ val workingResidualsR = Array(0.96, -0.48, -0.32, 0.24)
+ val responseResidualsR = Array(0.96, -0.48, -0.32, 0.24)
+ val seCoefR = Array(2.7782, 0.9798, 4.804)
+ val tValsR = Array(-0.34555, 1.73506, 1.15321)
+ val pValsR = Array(0.78819, 0.33286, 0.45478)
+ val dispersionR = 1.92
+ val nullDevianceR = 152.1
+ val residualDevianceR = 1.92
val residualDegreeOfFreedomNullR = 3
val residualDegreeOfFreedomR = 1
- val aicR = 18.783
+ val aicR = 13.23758
assert(model.hasSummary)
val summary = model.summary
@@ -912,7 +995,7 @@ class GeneralizedLinearRegressionSuite
assert(summary.aic ~== aicR absTol 1E-3)
assert(summary.solver === "irls")
- val summary2: GeneralizedLinearRegressionSummary = model.evaluate(datasetWithWeight)
+ val summary2: GeneralizedLinearRegressionSummary = model.evaluate(dataset)
assert(summary.predictions.columns.toSet === summary2.predictions.columns.toSet)
assert(summary.predictionCol === summary2.predictionCol)
assert(summary.rank === summary2.rank)
@@ -925,79 +1008,79 @@ class GeneralizedLinearRegressionSuite
assert(summary.aic === summary2.aic)
}
- test("glm summary: binomial family with weight") {
+ test("glm summary: binomial family with weight and offset") {
/*
- R code:
+ R code:
- A <- matrix(c(0, 1, 2, 3, 5, 2, 1, 3), 4, 2)
- b <- c(1, 0.5, 1, 0)
- w <- c(1, 2.0, 0.3, 4.7)
- df <- as.data.frame(cbind(A, b))
+ df <- as.data.frame(matrix(c(
+ 0.2, 1.0, 2.0, 0.0, 5.0,
+ 0.5, 2.1, 0.5, 1.0, 2.0,
+ 0.9, 0.4, 1.0, 2.0, 1.0,
+ 0.7, 0.7, 0.0, 3.0, 3.0), 4, 5, byrow = TRUE))
*/
- val datasetWithWeight = Seq(
- Instance(1.0, 1.0, Vectors.dense(0.0, 5.0).toSparse),
- Instance(0.5, 2.0, Vectors.dense(1.0, 2.0)),
- Instance(1.0, 0.3, Vectors.dense(2.0, 1.0)),
- Instance(0.0, 4.7, Vectors.dense(3.0, 3.0))
+ val dataset = Seq(
+ OffsetInstance(0.2, 1.0, 2.0, Vectors.dense(0.0, 5.0)),
+ OffsetInstance(0.5, 2.1, 0.5, Vectors.dense(1.0, 2.0)),
+ OffsetInstance(0.9, 0.4, 1.0, Vectors.dense(2.0, 1.0)),
+ OffsetInstance(0.7, 0.7, 0.0, Vectors.dense(3.0, 3.0))
).toDF()
-
/*
- R code:
-
- model <- glm(formula = "b ~ . -1", family="binomial", data = df, weights = w)
- summary(model)
-
- Deviance Residuals:
- 1 2 3 4
- 0.2404 0.1965 1.2824 -0.6916
+ R code:
- Coefficients:
- Estimate Std. Error z value Pr(>|z|)
- x1 -1.6901 1.2764 -1.324 0.185
- x2 0.7059 0.9449 0.747 0.455
+ model <- glm(formula = "V1 ~ V4 + V5", family = "binomial", data = df,
+ weights = V2, offset = V3)
+ summary(model)
- (Dispersion parameter for binomial family taken to be 1)
+ Deviance Residuals:
+ 1 2 3 4
+ 0.002584 -0.003800 0.012478 -0.001796
- Null deviance: 8.3178 on 4 degrees of freedom
- Residual deviance: 2.2193 on 2 degrees of freedom
- AIC: 5.9915
+ Coefficients:
+ Estimate Std. Error z value Pr(>|z|)
+ (Intercept) -0.2147 3.5687 -0.060 0.952
+ V4 0.9912 1.2344 0.803 0.422
+ V5 -0.6356 0.9669 -0.657 0.511
- Number of Fisher Scoring iterations: 5
+ (Dispersion parameter for binomial family taken to be 1)
- residuals(model, type="pearson")
- 1 2 3 4
- 0.171217 0.197406 2.085864 -0.495332
+ Null deviance: 2.17560881 on 3 degrees of freedom
+ Residual deviance: 0.00018005 on 1 degrees of freedom
+ AIC: 10.245
- residuals(model, type="working")
- 1 2 3 4
- 1.029315 0.281881 15.502768 -1.052203
+ Number of Fisher Scoring iterations: 4
- residuals(model, type="response")
- 1 2 3 4
- 0.028480 0.069123 0.935495 -0.049613
+ residuals(model, type = "pearson")
+ 1 2 3 4
+ 0.002586113 -0.003799744 0.012372235 -0.001796892
+ residuals(model, type = "working")
+ 1 2 3 4
+ 0.006477857 -0.005244163 0.063541250 -0.004691064
+ residuals(model, type = "response")
+ 1 2 3 4
+ 0.0010324375 -0.0013110318 0.0060225522 -0.0009832738
*/
val trainer = new GeneralizedLinearRegression()
.setFamily("Binomial")
.setWeightCol("weight")
- .setFitIntercept(false)
-
- val model = trainer.fit(datasetWithWeight)
-
- val coefficientsR = Vectors.dense(Array(-1.690134, 0.705929))
- val interceptR = 0.0
- val devianceResidualsR = Array(0.2404, 0.1965, 1.2824, -0.6916)
- val pearsonResidualsR = Array(0.171217, 0.197406, 2.085864, -0.495332)
- val workingResidualsR = Array(1.029315, 0.281881, 15.502768, -1.052203)
- val responseResidualsR = Array(0.02848, 0.069123, 0.935495, -0.049613)
- val seCoefR = Array(1.276417, 0.944934)
- val tValsR = Array(-1.324124, 0.747068)
- val pValsR = Array(0.185462, 0.455023)
- val dispersionR = 1.0
- val nullDevianceR = 8.3178
- val residualDevianceR = 2.2193
- val residualDegreeOfFreedomNullR = 4
- val residualDegreeOfFreedomR = 2
- val aicR = 5.991537
+ .setOffsetCol("offset")
+
+ val model = trainer.fit(dataset)
+
+ val coefficientsR = Vectors.dense(Array(0.99117, -0.63561))
+ val interceptR = -0.21471
+ val devianceResidualsR = Array(0.00258, -0.0038, 0.01248, -0.0018)
+ val pearsonResidualsR = Array(0.00259, -0.0038, 0.01237, -0.0018)
+ val workingResidualsR = Array(0.00648, -0.00524, 0.06354, -0.00469)
+ val responseResidualsR = Array(0.00103, -0.00131, 0.00602, -0.00098)
+ val seCoefR = Array(1.23439, 0.9669, 3.56866)
+ val tValsR = Array(0.80297, -0.65737, -0.06017)
+ val pValsR = Array(0.42199, 0.51094, 0.95202)
+ val dispersionR = 1
+ val nullDevianceR = 2.17561
+ val residualDevianceR = 0.00018
+ val residualDegreeOfFreedomNullR = 3
+ val residualDegreeOfFreedomR = 1
+ val aicR = 10.24453
val summary = model.summary
val devianceResiduals = summary.residuals()
@@ -1040,81 +1123,79 @@ class GeneralizedLinearRegressionSuite
assert(summary.solver === "irls")
}
- test("glm summary: poisson family with weight") {
+ test("glm summary: poisson family with weight and offset") {
/*
- R code:
+ R code:
- A <- matrix(c(0, 1, 2, 3, 5, 7, 11, 13), 4, 2)
- b <- c(2, 8, 3, 9)
- w <- c(1, 2, 3, 4)
- df <- as.data.frame(cbind(A, b))
+ A <- matrix(c(0, 1, 2, 3, 5, 7, 11, 13), 4, 2)
+ b <- c(2, 8, 3, 9)
+ w <- c(1, 2, 3, 4)
+ off <- c(2, 3, 1, 4)
+ df <- as.data.frame(cbind(A, b))
*/
- val datasetWithWeight = Seq(
- Instance(2.0, 1.0, Vectors.dense(0.0, 5.0).toSparse),
- Instance(8.0, 2.0, Vectors.dense(1.0, 7.0)),
- Instance(3.0, 3.0, Vectors.dense(2.0, 11.0)),
- Instance(9.0, 4.0, Vectors.dense(3.0, 13.0))
+ val dataset = Seq(
+ OffsetInstance(2.0, 1.0, 2.0, Vectors.dense(0.0, 5.0).toSparse),
+ OffsetInstance(8.0, 2.0, 3.0, Vectors.dense(1.0, 7.0)),
+ OffsetInstance(3.0, 3.0, 1.0, Vectors.dense(2.0, 11.0)),
+ OffsetInstance(9.0, 4.0, 4.0, Vectors.dense(3.0, 13.0))
).toDF()
/*
- R code:
-
- model <- glm(formula = "b ~ .", family="poisson", data = df, weights = w)
- summary(model)
-
- Deviance Residuals:
- 1 2 3 4
- -0.28952 0.11048 0.14839 -0.07268
-
- Coefficients:
- Estimate Std. Error z value Pr(>|z|)
- (Intercept) 6.2999 1.6086 3.916 8.99e-05 ***
- V1 3.3241 1.0184 3.264 0.00110 **
- V2 -1.0818 0.3522 -3.071 0.00213 **
- ---
- Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
-
- (Dispersion parameter for poisson family taken to be 1)
-
- Null deviance: 15.38066 on 3 degrees of freedom
- Residual deviance: 0.12333 on 1 degrees of freedom
- AIC: 41.803
-
- Number of Fisher Scoring iterations: 3
+ R code:
- residuals(model, type="pearson")
- 1 2 3 4
- -0.28043145 0.11099310 0.14963714 -0.07253611
+ model <- glm(formula = "b ~ .", family = "poisson", data = df,
+ weights = w, offset = off)
+ summary(model)
- residuals(model, type="working")
- 1 2 3 4
- -0.17960679 0.02813593 0.05113852 -0.01201650
+ Deviance Residuals:
+ 1 2 3 4
+ -2.0480 1.2315 1.8293 -0.7107
- residuals(model, type="response")
- 1 2 3 4
- -0.4378554 0.2189277 0.1459518 -0.1094638
+ Coefficients:
+ Estimate Std. Error z value Pr(>|z|)
+ (Intercept) -4.5678 1.9625 -2.328 0.0199
+ V1 -2.8784 1.1683 -2.464 0.0137
+ V2 0.8859 0.4170 2.124 0.0336
+
+ (Dispersion parameter for poisson family taken to be 1)
+
+ Null deviance: 22.5585 on 3 degrees of freedom
+ Residual deviance: 9.5622 on 1 degrees of freedom
+ AIC: 51.242
+
+ Number of Fisher Scoring iterations: 5
+
+ residuals(model, type = "pearson")
+ 1 2 3 4
+ -1.7480418 1.3037611 2.0750099 -0.6972966
+ residuals(model, type = "working")
+ 1 2 3 4
+ -0.6891489 0.3833588 0.9710682 -0.1096590
+ residuals(model, type = "response")
+ 1 2 3 4
+ -4.433948 2.216974 1.477983 -1.108487
*/
val trainer = new GeneralizedLinearRegression()
.setFamily("Poisson")
.setWeightCol("weight")
- .setFitIntercept(true)
-
- val model = trainer.fit(datasetWithWeight)
-
- val coefficientsR = Vectors.dense(Array(3.3241, -1.0818))
- val interceptR = 6.2999
- val devianceResidualsR = Array(-0.28952, 0.11048, 0.14839, -0.07268)
- val pearsonResidualsR = Array(-0.28043145, 0.11099310, 0.14963714, -0.07253611)
- val workingResidualsR = Array(-0.17960679, 0.02813593, 0.05113852, -0.01201650)
- val responseResidualsR = Array(-0.4378554, 0.2189277, 0.1459518, -0.1094638)
- val seCoefR = Array(1.0184, 0.3522, 1.6086)
- val tValsR = Array(3.264, -3.071, 3.916)
- val pValsR = Array(0.00110, 0.00213, 0.00009)
- val dispersionR = 1.0
- val nullDevianceR = 15.38066
- val residualDevianceR = 0.12333
+ .setOffsetCol("offset")
+
+ val model = trainer.fit(dataset)
+
+ val coefficientsR = Vectors.dense(Array(-2.87843, 0.88589))
+ val interceptR = -4.56784
+ val devianceResidualsR = Array(-2.04796, 1.23149, 1.82933, -0.71066)
+ val pearsonResidualsR = Array(-1.74804, 1.30376, 2.07501, -0.6973)
+ val workingResidualsR = Array(-0.68915, 0.38336, 0.97107, -0.10966)
+ val responseResidualsR = Array(-4.43395, 2.21697, 1.47798, -1.10849)
+ val seCoefR = Array(1.16826, 0.41703, 1.96249)
+ val tValsR = Array(-2.46387, 2.12428, -2.32757)
+ val pValsR = Array(0.01374, 0.03365, 0.01993)
+ val dispersionR = 1
+ val nullDevianceR = 22.55853
+ val residualDevianceR = 9.5622
val residualDegreeOfFreedomNullR = 3
val residualDegreeOfFreedomR = 1
- val aicR = 41.803
+ val aicR = 51.24218
val summary = model.summary
val devianceResiduals = summary.residuals()
@@ -1157,78 +1238,79 @@ class GeneralizedLinearRegressionSuite
assert(summary.solver === "irls")
}
- test("glm summary: gamma family with weight") {
+ test("glm summary: gamma family with weight and offset") {
/*
- R code:
+ R code:
- A <- matrix(c(0, 1, 2, 3, 5, 7, 11, 13), 4, 2)
- b <- c(2, 8, 3, 9)
- w <- c(1, 2, 3, 4)
- df <- as.data.frame(cbind(A, b))
+ A <- matrix(c(0, 5, 1, 2, 2, 1, 3, 3), 4, 2, byrow = TRUE)
+ b <- c(1, 2, 1, 2)
+ w <- c(1, 2, 3, 4)
+ off <- c(0, 0.5, 1, 0)
+ df <- as.data.frame(cbind(A, b))
*/
- val datasetWithWeight = Seq(
- Instance(2.0, 1.0, Vectors.dense(0.0, 5.0).toSparse),
- Instance(8.0, 2.0, Vectors.dense(1.0, 7.0)),
- Instance(3.0, 3.0, Vectors.dense(2.0, 11.0)),
- Instance(9.0, 4.0, Vectors.dense(3.0, 13.0))
+ val dataset = Seq(
+ OffsetInstance(1.0, 1.0, 0.0, Vectors.dense(0.0, 5.0)),
+ OffsetInstance(2.0, 2.0, 0.5, Vectors.dense(1.0, 2.0)),
+ OffsetInstance(1.0, 3.0, 1.0, Vectors.dense(2.0, 1.0)),
+ OffsetInstance(2.0, 4.0, 0.0, Vectors.dense(3.0, 3.0))
).toDF()
/*
- R code:
-
- model <- glm(formula = "b ~ .", family="Gamma", data = df, weights = w)
- summary(model)
+ R code:
- Deviance Residuals:
- 1 2 3 4
- -0.26343 0.05761 0.12818 -0.03484
+ model <- glm(formula = "b ~ .", family = "Gamma", data = df,
+ weights = w, offset = off)
+ summary(model)
- Coefficients:
- Estimate Std. Error t value Pr(>|t|)
- (Intercept) -0.81511 0.23449 -3.476 0.178
- V1 -0.72730 0.16137 -4.507 0.139
- V2 0.23894 0.05481 4.359 0.144
+ Deviance Residuals:
+ 1 2 3 4
+ -0.17095 0.19867 -0.23604 0.03241
- (Dispersion parameter for Gamma family taken to be 0.07986091)
+ Coefficients:
+ Estimate Std. Error t value Pr(>|t|)
+ (Intercept) -0.56474 0.23866 -2.366 0.255
+ V1 0.07695 0.06931 1.110 0.467
+ V2 0.28068 0.07320 3.835 0.162
- Null deviance: 2.937462 on 3 degrees of freedom
- Residual deviance: 0.090358 on 1 degrees of freedom
- AIC: 23.202
+ (Dispersion parameter for Gamma family taken to be 0.1212174)
- Number of Fisher Scoring iterations: 4
+ Null deviance: 2.02568 on 3 degrees of freedom
+ Residual deviance: 0.12546 on 1 degrees of freedom
+ AIC: 0.93388
- residuals(model, type="pearson")
- 1 2 3 4
- -0.24082508 0.05839241 0.13135766 -0.03463621
+ Number of Fisher Scoring iterations: 4
- residuals(model, type="working")
+ residuals(model, type = "pearson")
+ 1 2 3 4
+ -0.16134949 0.20807694 -0.22544551 0.03258777
+ residuals(model, type = "working")
1 2 3 4
- 0.091414181 -0.005374314 -0.027196998 0.001890910
-
- residuals(model, type="response")
- 1 2 3 4
- -0.6344390 0.3172195 0.2114797 -0.1586097
+ 0.135315831 -0.084390309 0.113219135 -0.008279688
+ residuals(model, type = "response")
+ 1 2 3 4
+ -0.1923918 0.2565224 -0.1496381 0.0320653
*/
val trainer = new GeneralizedLinearRegression()
.setFamily("Gamma")
.setWeightCol("weight")
+ .setOffsetCol("offset")
+
+ val model = trainer.fit(dataset)
- val model = trainer.fit(datasetWithWeight)
-
- val coefficientsR = Vectors.dense(Array(-0.72730, 0.23894))
- val interceptR = -0.81511
- val devianceResidualsR = Array(-0.26343, 0.05761, 0.12818, -0.03484)
- val pearsonResidualsR = Array(-0.24082508, 0.05839241, 0.13135766, -0.03463621)
- val workingResidualsR = Array(0.091414181, -0.005374314, -0.027196998, 0.001890910)
- val responseResidualsR = Array(-0.6344390, 0.3172195, 0.2114797, -0.1586097)
- val seCoefR = Array(0.16137, 0.05481, 0.23449)
- val tValsR = Array(-4.507, 4.359, -3.476)
- val pValsR = Array(0.139, 0.144, 0.178)
- val dispersionR = 0.07986091
- val nullDevianceR = 2.937462
- val residualDevianceR = 0.090358
+ val coefficientsR = Vectors.dense(Array(0.07695, 0.28068))
+ val interceptR = -0.56474
+ val devianceResidualsR = Array(-0.17095, 0.19867, -0.23604, 0.03241)
+ val pearsonResidualsR = Array(-0.16135, 0.20808, -0.22545, 0.03259)
+ val workingResidualsR = Array(0.13532, -0.08439, 0.11322, -0.00828)
+ val responseResidualsR = Array(-0.19239, 0.25652, -0.14964, 0.03207)
+ val seCoefR = Array(0.06931, 0.0732, 0.23866)
+ val tValsR = Array(1.11031, 3.83453, -2.3663)
+ val pValsR = Array(0.46675, 0.16241, 0.25454)
+ val dispersionR = 0.12122
+ val nullDevianceR = 2.02568
+ val residualDevianceR = 0.12546
val residualDegreeOfFreedomNullR = 3
val residualDegreeOfFreedomR = 1
- val aicR = 23.202
+ val aicR = 0.93388
val summary = model.summary
val devianceResiduals = summary.residuals()
@@ -1271,77 +1353,81 @@ class GeneralizedLinearRegressionSuite
assert(summary.solver === "irls")
}
- test("glm summary: tweedie family with weight") {
+ test("glm summary: tweedie family with weight and offset") {
/*
R code:
- library(statmod)
df <- as.data.frame(matrix(c(
- 1.0, 1.0, 0.0, 5.0,
- 0.5, 2.0, 1.0, 2.0,
- 1.0, 3.0, 2.0, 1.0,
- 0.0, 4.0, 3.0, 3.0), 4, 4, byrow = TRUE))
+ 1.0, 1.0, 1.0, 0.0, 5.0,
+ 0.5, 2.0, 3.0, 1.0, 2.0,
+ 1.0, 3.0, 2.0, 2.0, 1.0,
+ 0.0, 4.0, 0.0, 3.0, 3.0), 4, 5, byrow = TRUE))
+ */
+ val dataset = Seq(
+ OffsetInstance(1.0, 1.0, 1.0, Vectors.dense(0.0, 5.0)),
+ OffsetInstance(0.5, 2.0, 3.0, Vectors.dense(1.0, 2.0)),
+ OffsetInstance(1.0, 3.0, 2.0, Vectors.dense(2.0, 1.0)),
+ OffsetInstance(0.0, 4.0, 0.0, Vectors.dense(3.0, 3.0))
+ ).toDF()
+ /*
+ R code:
- model <- glm(V1 ~ -1 + V3 + V4, data = df, weights = V2,
- family = tweedie(var.power = 1.6, link.power = 0))
+ library(statmod)
+ model <- glm(V1 ~ V4 + V5, data = df, weights = V2, offset = V3,
+ family = tweedie(var.power = 1.6, link.power = 0.0))
summary(model)
Deviance Residuals:
1 2 3 4
- 0.6210 -0.0515 1.6935 -3.2539
+ 0.8917 -2.1396 1.2252 -1.7946
Coefficients:
- Estimate Std. Error t value Pr(>|t|)
- V3 -0.4087 0.5205 -0.785 0.515
- V4 -0.1212 0.4082 -0.297 0.794
+ Estimate Std. Error t value Pr(>|t|)
+ (Intercept) -0.03047 3.65000 -0.008 0.995
+ V4 -1.14577 1.41674 -0.809 0.567
+ V5 -0.36585 0.97065 -0.377 0.771
- (Dispersion parameter for Tweedie family taken to be 3.830036)
+ (Dispersion parameter for Tweedie family taken to be 6.334961)
- Null deviance: 20.702 on 4 degrees of freedom
- Residual deviance: 13.844 on 2 degrees of freedom
+ Null deviance: 12.784 on 3 degrees of freedom
+ Residual deviance: 10.095 on 1 degrees of freedom
AIC: NA
- Number of Fisher Scoring iterations: 11
-
- residuals(model, type="pearson")
- 1 2 3 4
- 0.7383616 -0.0509458 2.2348337 -1.4552090
- residuals(model, type="working")
- 1 2 3 4
- 0.83354150 -0.04103552 1.55676369 -1.00000000
- residuals(model, type="response")
- 1 2 3 4
- 0.45460738 -0.02139574 0.60888055 -0.20392801
+ Number of Fisher Scoring iterations: 18
+
+ residuals(model, type = "pearson")
+ 1 2 3 4
+ 1.1472554 -1.4642569 1.4935199 -0.8025842
+ residuals(model, type = "working")
+ 1 2 3 4
+ 1.3624928 -0.8322375 0.9894580 -1.0000000
+ residuals(model, type = "response")
+ 1 2 3 4
+ 0.57671828 -2.48040354 0.49735052 -0.01040646
*/
- val datasetWithWeight = Seq(
- Instance(1.0, 1.0, Vectors.dense(0.0, 5.0)),
- Instance(0.5, 2.0, Vectors.dense(1.0, 2.0)),
- Instance(1.0, 3.0, Vectors.dense(2.0, 1.0)),
- Instance(0.0, 4.0, Vectors.dense(3.0, 3.0))
- ).toDF()
-
val trainer = new GeneralizedLinearRegression()
.setFamily("tweedie")
.setVariancePower(1.6)
.setLinkPower(0.0)
.setWeightCol("weight")
- .setFitIntercept(false)
-
- val model = trainer.fit(datasetWithWeight)
- val coefficientsR = Vectors.dense(Array(-0.408746, -0.12125))
- val interceptR = 0.0
- val devianceResidualsR = Array(0.621047, -0.051515, 1.693473, -3.253946)
- val pearsonResidualsR = Array(0.738362, -0.050946, 2.234834, -1.455209)
- val workingResidualsR = Array(0.833541, -0.041036, 1.556764, -1.0)
- val responseResidualsR = Array(0.454607, -0.021396, 0.608881, -0.203928)
- val seCoefR = Array(0.520519, 0.408215)
- val tValsR = Array(-0.785267, -0.297024)
- val pValsR = Array(0.514549, 0.794457)
- val dispersionR = 3.830036
- val nullDevianceR = 20.702
- val residualDevianceR = 13.844
- val residualDegreeOfFreedomNullR = 4
- val residualDegreeOfFreedomR = 2
+ .setOffsetCol("offset")
+
+ val model = trainer.fit(dataset)
+
+ val coefficientsR = Vectors.dense(Array(-1.14577, -0.36585))
+ val interceptR = -0.03047
+ val devianceResidualsR = Array(0.89171, -2.13961, 1.2252, -1.79463)
+ val pearsonResidualsR = Array(1.14726, -1.46426, 1.49352, -0.80258)
+ val workingResidualsR = Array(1.36249, -0.83224, 0.98946, -1)
+ val responseResidualsR = Array(0.57672, -2.4804, 0.49735, -0.01041)
+ val seCoefR = Array(1.41674, 0.97065, 3.65)
+ val tValsR = Array(-0.80873, -0.37691, -0.00835)
+ val pValsR = Array(0.56707, 0.77053, 0.99468)
+ val dispersionR = 6.33496
+ val nullDevianceR = 12.78358
+ val residualDevianceR = 10.09488
+ val residualDegreeOfFreedomNullR = 3
+ val residualDegreeOfFreedomR = 1
val summary = model.summary
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org