You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by db...@apache.org on 2017/04/27 20:48:55 UTC
spark git commit: [SPARK-20047][ML] Constrained Logistic Regression
Repository: spark
Updated Branches:
refs/heads/master 039e32ca1 -> 606432a13
[SPARK-20047][ML] Constrained Logistic Regression
## What changes were proposed in this pull request?
MLlib ```LogisticRegression``` should support bound constrained optimization (only for L2 regularization). Users can add bound constraints to coefficients to make the solver produce solution in the specified range.
Under the hood, we call Breeze [```L-BFGS-B```](https://github.com/scalanlp/breeze/blob/master/math/src/main/scala/breeze/optimize/LBFGSB.scala) as the solver for bound constrained optimization. But in the current breeze implementation, there are some bugs in L-BFGS-B, and https://github.com/scalanlp/breeze/pull/633 fixed them. We need to upgrade dependent breeze later, and currently we use the workaround L-BFGS-B in this PR temporary for reviewing.
## How was this patch tested?
Unit tests.
Author: Yanbo Liang <yb...@gmail.com>
Closes #17715 from yanboliang/spark-20047.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/606432a1
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/606432a1
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/606432a1
Branch: refs/heads/master
Commit: 606432a13ad22d862c7cb5028ad6fe73c9985423
Parents: 039e32c
Author: Yanbo Liang <yb...@gmail.com>
Authored: Thu Apr 27 20:48:43 2017 +0000
Committer: DB Tsai <db...@dbtsai.com>
Committed: Thu Apr 27 20:48:43 2017 +0000
----------------------------------------------------------------------
.../ml/classification/LogisticRegression.scala | 223 ++++++++-
.../LogisticRegressionSuite.scala | 466 ++++++++++++++++++-
2 files changed, 682 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/606432a1/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
index 44b3478..d7dde32 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
@@ -22,7 +22,7 @@ import java.util.Locale
import scala.collection.mutable
import breeze.linalg.{DenseVector => BDV}
-import breeze.optimize.{CachedDiffFunction, DiffFunction, LBFGS => BreezeLBFGS, OWLQN => BreezeOWLQN}
+import breeze.optimize.{CachedDiffFunction, DiffFunction, LBFGS => BreezeLBFGS, LBFGSB => BreezeLBFGSB, OWLQN => BreezeOWLQN}
import org.apache.hadoop.fs.Path
import org.apache.spark.SparkException
@@ -178,11 +178,86 @@ private[classification] trait LogisticRegressionParams extends ProbabilisticClas
}
}
+ /**
+ * The lower bounds on coefficients if fitting under bound constrained optimization.
+ * The bound matrix must be compatible with the shape (1, number of features) for binomial
+ * regression, or (number of classes, number of features) for multinomial regression.
+ * Otherwise, it throws exception.
+ *
+ * @group param
+ */
+ @Since("2.2.0")
+ val lowerBoundsOnCoefficients: Param[Matrix] = new Param(this, "lowerBoundsOnCoefficients",
+ "The lower bounds on coefficients if fitting under bound constrained optimization.")
+
+ /** @group getParam */
+ @Since("2.2.0")
+ def getLowerBoundsOnCoefficients: Matrix = $(lowerBoundsOnCoefficients)
+
+ /**
+ * The upper bounds on coefficients if fitting under bound constrained optimization.
+ * The bound matrix must be compatible with the shape (1, number of features) for binomial
+ * regression, or (number of classes, number of features) for multinomial regression.
+ * Otherwise, it throws exception.
+ *
+ * @group param
+ */
+ @Since("2.2.0")
+ val upperBoundsOnCoefficients: Param[Matrix] = new Param(this, "upperBoundsOnCoefficients",
+ "The upper bounds on coefficients if fitting under bound constrained optimization.")
+
+ /** @group getParam */
+ @Since("2.2.0")
+ def getUpperBoundsOnCoefficients: Matrix = $(upperBoundsOnCoefficients)
+
+ /**
+ * The lower bounds on intercepts if fitting under bound constrained optimization.
+ * The bounds vector size must be equal with 1 for binomial regression, or the number
+ * of classes for multinomial regression. Otherwise, it throws exception.
+ *
+ * @group param
+ */
+ @Since("2.2.0")
+ val lowerBoundsOnIntercepts: Param[Vector] = new Param(this, "lowerBoundsOnIntercepts",
+ "The lower bounds on intercepts if fitting under bound constrained optimization.")
+
+ /** @group getParam */
+ @Since("2.2.0")
+ def getLowerBoundsOnIntercepts: Vector = $(lowerBoundsOnIntercepts)
+
+ /**
+ * The upper bounds on intercepts if fitting under bound constrained optimization.
+ * The bound vector size must be equal with 1 for binomial regression, or the number
+ * of classes for multinomial regression. Otherwise, it throws exception.
+ *
+ * @group param
+ */
+ @Since("2.2.0")
+ val upperBoundsOnIntercepts: Param[Vector] = new Param(this, "upperBoundsOnIntercepts",
+ "The upper bounds on intercepts if fitting under bound constrained optimization.")
+
+ /** @group getParam */
+ @Since("2.2.0")
+ def getUpperBoundsOnIntercepts: Vector = $(upperBoundsOnIntercepts)
+
+ protected def usingBoundConstrainedOptimization: Boolean = {
+ isSet(lowerBoundsOnCoefficients) || isSet(upperBoundsOnCoefficients) ||
+ isSet(lowerBoundsOnIntercepts) || isSet(upperBoundsOnIntercepts)
+ }
+
override protected def validateAndTransformSchema(
schema: StructType,
fitting: Boolean,
featuresDataType: DataType): StructType = {
checkThresholdConsistency()
+ if (usingBoundConstrainedOptimization) {
+ require($(elasticNetParam) == 0.0, "Fitting under bound constrained optimization only " +
+ s"supports L2 regularization, but got elasticNetParam = $getElasticNetParam.")
+ }
+ if (!$(fitIntercept)) {
+ require(!isSet(lowerBoundsOnIntercepts) && !isSet(upperBoundsOnIntercepts),
+ "Pls don't set bounds on intercepts if fitting without intercept.")
+ }
super.validateAndTransformSchema(schema, fitting, featuresDataType)
}
}
@@ -217,6 +292,9 @@ class LogisticRegression @Since("1.2.0") (
* For alpha in (0,1), the penalty is a combination of L1 and L2.
* Default is 0.0 which is an L2 penalty.
*
+ * Note: Fitting under bound constrained optimization only supports L2 regularization,
+ * so throws exception if this param is non-zero value.
+ *
* @group setParam
*/
@Since("1.4.0")
@@ -312,6 +390,71 @@ class LogisticRegression @Since("1.2.0") (
def setAggregationDepth(value: Int): this.type = set(aggregationDepth, value)
setDefault(aggregationDepth -> 2)
+ /**
+ * Set the lower bounds on coefficients if fitting under bound constrained optimization.
+ *
+ * @group setParam
+ */
+ @Since("2.2.0")
+ def setLowerBoundsOnCoefficients(value: Matrix): this.type = set(lowerBoundsOnCoefficients, value)
+
+ /**
+ * Set the upper bounds on coefficients if fitting under bound constrained optimization.
+ *
+ * @group setParam
+ */
+ @Since("2.2.0")
+ def setUpperBoundsOnCoefficients(value: Matrix): this.type = set(upperBoundsOnCoefficients, value)
+
+ /**
+ * Set the lower bounds on intercepts if fitting under bound constrained optimization.
+ *
+ * @group setParam
+ */
+ @Since("2.2.0")
+ def setLowerBoundsOnIntercepts(value: Vector): this.type = set(lowerBoundsOnIntercepts, value)
+
+ /**
+ * Set the upper bounds on intercepts if fitting under bound constrained optimization.
+ *
+ * @group setParam
+ */
+ @Since("2.2.0")
+ def setUpperBoundsOnIntercepts(value: Vector): this.type = set(upperBoundsOnIntercepts, value)
+
+ private def assertBoundConstrainedOptimizationParamsValid(
+ numCoefficientSets: Int,
+ numFeatures: Int): Unit = {
+ if (isSet(lowerBoundsOnCoefficients)) {
+ require($(lowerBoundsOnCoefficients).numRows == numCoefficientSets &&
+ $(lowerBoundsOnCoefficients).numCols == numFeatures)
+ }
+ if (isSet(upperBoundsOnCoefficients)) {
+ require($(upperBoundsOnCoefficients).numRows == numCoefficientSets &&
+ $(upperBoundsOnCoefficients).numCols == numFeatures)
+ }
+ if (isSet(lowerBoundsOnIntercepts)) {
+ require($(lowerBoundsOnIntercepts).size == numCoefficientSets)
+ }
+ if (isSet(upperBoundsOnIntercepts)) {
+ require($(upperBoundsOnIntercepts).size == numCoefficientSets)
+ }
+ if (isSet(lowerBoundsOnCoefficients) && isSet(upperBoundsOnCoefficients)) {
+ require($(lowerBoundsOnCoefficients).toArray.zip($(upperBoundsOnCoefficients).toArray)
+ .forall(x => x._1 <= x._2), "LowerBoundsOnCoefficients should always " +
+ "less than or equal to upperBoundsOnCoefficients, but found: " +
+ s"lowerBoundsOnCoefficients = $getLowerBoundsOnCoefficients, " +
+ s"upperBoundsOnCoefficients = $getUpperBoundsOnCoefficients.")
+ }
+ if (isSet(lowerBoundsOnIntercepts) && isSet(upperBoundsOnIntercepts)) {
+ require($(lowerBoundsOnIntercepts).toArray.zip($(upperBoundsOnIntercepts).toArray)
+ .forall(x => x._1 <= x._2), "LowerBoundsOnIntercepts should always " +
+ "less than or equal to upperBoundsOnIntercepts, but found: " +
+ s"lowerBoundsOnIntercepts = $getLowerBoundsOnIntercepts, " +
+ s"upperBoundsOnIntercepts = $getUpperBoundsOnIntercepts.")
+ }
+ }
+
private var optInitialModel: Option[LogisticRegressionModel] = None
private[spark] def setInitialModel(model: LogisticRegressionModel): this.type = {
@@ -378,6 +521,11 @@ class LogisticRegression @Since("1.2.0") (
}
val numCoefficientSets = if (isMultinomial) numClasses else 1
+ // Check params interaction is valid if fitting under bound constrained optimization.
+ if (usingBoundConstrainedOptimization) {
+ assertBoundConstrainedOptimizationParamsValid(numCoefficientSets, numFeatures)
+ }
+
if (isDefined(thresholds)) {
require($(thresholds).length == numClasses, this.getClass.getSimpleName +
".train() called with non-matching numClasses and thresholds.length." +
@@ -397,7 +545,7 @@ class LogisticRegression @Since("1.2.0") (
val isConstantLabel = histogram.count(_ != 0.0) == 1
- if ($(fitIntercept) && isConstantLabel) {
+ if ($(fitIntercept) && isConstantLabel && !usingBoundConstrainedOptimization) {
logWarning(s"All labels are the same value and fitIntercept=true, so the coefficients " +
s"will be zeros. Training is not needed.")
val constantLabelIndex = Vectors.dense(histogram).argmax
@@ -434,8 +582,53 @@ class LogisticRegression @Since("1.2.0") (
$(standardization), bcFeaturesStd, regParamL2, multinomial = isMultinomial,
$(aggregationDepth))
+ val numCoeffsPlusIntercepts = numFeaturesPlusIntercept * numCoefficientSets
+
+ val (lowerBounds, upperBounds): (Array[Double], Array[Double]) = {
+ if (usingBoundConstrainedOptimization) {
+ val lowerBounds = Array.fill[Double](numCoeffsPlusIntercepts)(Double.NegativeInfinity)
+ val upperBounds = Array.fill[Double](numCoeffsPlusIntercepts)(Double.PositiveInfinity)
+ val isSetLowerBoundsOnCoefficients = isSet(lowerBoundsOnCoefficients)
+ val isSetUpperBoundsOnCoefficients = isSet(upperBoundsOnCoefficients)
+ val isSetLowerBoundsOnIntercepts = isSet(lowerBoundsOnIntercepts)
+ val isSetUpperBoundsOnIntercepts = isSet(upperBoundsOnIntercepts)
+
+ var i = 0
+ while (i < numCoeffsPlusIntercepts) {
+ val coefficientSetIndex = i % numCoefficientSets
+ val featureIndex = i / numCoefficientSets
+ if (featureIndex < numFeatures) {
+ if (isSetLowerBoundsOnCoefficients) {
+ lowerBounds(i) = $(lowerBoundsOnCoefficients)(
+ coefficientSetIndex, featureIndex) * featuresStd(featureIndex)
+ }
+ if (isSetUpperBoundsOnCoefficients) {
+ upperBounds(i) = $(upperBoundsOnCoefficients)(
+ coefficientSetIndex, featureIndex) * featuresStd(featureIndex)
+ }
+ } else {
+ if (isSetLowerBoundsOnIntercepts) {
+ lowerBounds(i) = $(lowerBoundsOnIntercepts)(coefficientSetIndex)
+ }
+ if (isSetUpperBoundsOnIntercepts) {
+ upperBounds(i) = $(upperBoundsOnIntercepts)(coefficientSetIndex)
+ }
+ }
+ i += 1
+ }
+ (lowerBounds, upperBounds)
+ } else {
+ (null, null)
+ }
+ }
+
val optimizer = if ($(elasticNetParam) == 0.0 || $(regParam) == 0.0) {
- new BreezeLBFGS[BDV[Double]]($(maxIter), 10, $(tol))
+ if (lowerBounds != null && upperBounds != null) {
+ new BreezeLBFGSB(
+ BDV[Double](lowerBounds), BDV[Double](upperBounds), $(maxIter), 10, $(tol))
+ } else {
+ new BreezeLBFGS[BDV[Double]]($(maxIter), 10, $(tol))
+ }
} else {
val standardizationParam = $(standardization)
def regParamL1Fun = (index: Int) => {
@@ -546,6 +739,26 @@ class LogisticRegression @Since("1.2.0") (
math.log(histogram(1) / histogram(0)))
}
+ if (usingBoundConstrainedOptimization) {
+ // Make sure all initial values locate in the corresponding bound.
+ var i = 0
+ while (i < numCoeffsPlusIntercepts) {
+ val coefficientSetIndex = i % numCoefficientSets
+ val featureIndex = i / numCoefficientSets
+ if (initialCoefWithInterceptMatrix(coefficientSetIndex, featureIndex) < lowerBounds(i))
+ {
+ initialCoefWithInterceptMatrix.update(
+ coefficientSetIndex, featureIndex, lowerBounds(i))
+ } else if (
+ initialCoefWithInterceptMatrix(coefficientSetIndex, featureIndex) > upperBounds(i))
+ {
+ initialCoefWithInterceptMatrix.update(
+ coefficientSetIndex, featureIndex, upperBounds(i))
+ }
+ i += 1
+ }
+ }
+
val states = optimizer.iterations(new CachedDiffFunction(costFun),
new BDV[Double](initialCoefWithInterceptMatrix.toArray))
@@ -599,7 +812,7 @@ class LogisticRegression @Since("1.2.0") (
if (isIntercept) interceptVec.toArray(classIndex) = value
}
- if ($(regParam) == 0.0 && isMultinomial) {
+ if ($(regParam) == 0.0 && isMultinomial && !usingBoundConstrainedOptimization) {
/*
When no regularization is applied, the multinomial coefficients lack identifiability
because we do not use a pivot class. We can add any constant value to the coefficients
@@ -620,7 +833,7 @@ class LogisticRegression @Since("1.2.0") (
}
// center the intercepts when using multinomial algorithm
- if ($(fitIntercept) && isMultinomial) {
+ if ($(fitIntercept) && isMultinomial && !usingBoundConstrainedOptimization) {
val interceptArray = interceptVec.toArray
val interceptMean = interceptArray.sum / interceptArray.length
(0 until interceptVec.size).foreach { i => interceptArray(i) -= interceptMean }
http://git-wip-us.apache.org/repos/asf/spark/blob/606432a1/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
----------------------------------------------------------------------
diff --git a/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
index 83f575e..bf6bfe3 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
@@ -26,7 +26,7 @@ import org.apache.spark.{SparkException, SparkFunSuite}
import org.apache.spark.ml.attribute.NominalAttribute
import org.apache.spark.ml.classification.LogisticRegressionSuite._
import org.apache.spark.ml.feature.{Instance, LabeledPoint}
-import org.apache.spark.ml.linalg.{DenseMatrix, Matrices, SparseMatrix, Vector, Vectors}
+import org.apache.spark.ml.linalg.{DenseMatrix, Matrices, Matrix, SparseMatrix, Vector, Vectors}
import org.apache.spark.ml.param.{ParamMap, ParamsSuite}
import org.apache.spark.ml.util.{DefaultReadWriteTest, MLTestingUtils}
import org.apache.spark.ml.util.TestingUtils._
@@ -150,6 +150,54 @@ class LogisticRegressionSuite
assert(!model.hasSummary)
}
+ test("logistic regression: illegal params") {
+ val lowerBoundsOnCoefficients = Matrices.dense(1, 4, Array(1.0, 0.0, 1.0, 0.0))
+ val upperBoundsOnCoefficients1 = Matrices.dense(1, 4, Array(0.0, 1.0, 1.0, 0.0))
+ val upperBoundsOnCoefficients2 = Matrices.dense(1, 3, Array(1.0, 0.0, 1.0))
+ val lowerBoundsOnIntercepts = Vectors.dense(1.0)
+
+ // Work well when only set bound in one side.
+ new LogisticRegression()
+ .setLowerBoundsOnCoefficients(lowerBoundsOnCoefficients)
+ .fit(binaryDataset)
+
+ withClue("bound constrained optimization only supports L2 regularization") {
+ intercept[IllegalArgumentException] {
+ new LogisticRegression()
+ .setLowerBoundsOnCoefficients(lowerBoundsOnCoefficients)
+ .setElasticNetParam(1.0)
+ .fit(binaryDataset)
+ }
+ }
+
+ withClue("lowerBoundsOnCoefficients should less than or equal to upperBoundsOnCoefficients") {
+ intercept[IllegalArgumentException] {
+ new LogisticRegression()
+ .setLowerBoundsOnCoefficients(lowerBoundsOnCoefficients)
+ .setUpperBoundsOnCoefficients(upperBoundsOnCoefficients1)
+ .fit(binaryDataset)
+ }
+ }
+
+ withClue("the coefficients bound matrix mismatched with shape (1, number of features)") {
+ intercept[IllegalArgumentException] {
+ new LogisticRegression()
+ .setLowerBoundsOnCoefficients(lowerBoundsOnCoefficients)
+ .setUpperBoundsOnCoefficients(upperBoundsOnCoefficients2)
+ .fit(binaryDataset)
+ }
+ }
+
+ withClue("bounds on intercepts should not be set if fitting without intercept") {
+ intercept[IllegalArgumentException] {
+ new LogisticRegression()
+ .setLowerBoundsOnIntercepts(lowerBoundsOnIntercepts)
+ .setFitIntercept(false)
+ .fit(binaryDataset)
+ }
+ }
+ }
+
test("empty probabilityCol") {
val lr = new LogisticRegression().setProbabilityCol("")
val model = lr.fit(smallBinaryDataset)
@@ -610,6 +658,107 @@ class LogisticRegressionSuite
assert(model2.coefficients ~= coefficientsR relTol 1E-3)
}
+ test("binary logistic regression with intercept without regularization with bound") {
+ // Bound constrained optimization with bound on one side.
+ val upperBoundsOnCoefficients = Matrices.dense(1, 4, Array(1.0, 0.0, 1.0, 0.0))
+ val upperBoundsOnIntercepts = Vectors.dense(1.0)
+
+ val trainer1 = new LogisticRegression()
+ .setUpperBoundsOnCoefficients(upperBoundsOnCoefficients)
+ .setUpperBoundsOnIntercepts(upperBoundsOnIntercepts)
+ .setFitIntercept(true)
+ .setStandardization(true)
+ .setWeightCol("weight")
+ val trainer2 = new LogisticRegression()
+ .setUpperBoundsOnCoefficients(upperBoundsOnCoefficients)
+ .setUpperBoundsOnIntercepts(upperBoundsOnIntercepts)
+ .setFitIntercept(true)
+ .setStandardization(false)
+ .setWeightCol("weight")
+
+ val model1 = trainer1.fit(binaryDataset)
+ val model2 = trainer2.fit(binaryDataset)
+
+ // The solution is generated by https://github.com/yanboliang/bound-optimization.
+ val coefficientsExpected1 = Vectors.dense(0.06079437, 0.0, -0.26351059, -0.59102199)
+ val interceptExpected1 = 1.0
+
+ assert(model1.intercept ~== interceptExpected1 relTol 1E-3)
+ assert(model1.coefficients ~= coefficientsExpected1 relTol 1E-3)
+
+ // Without regularization, with or without standardization will converge to the same solution.
+ assert(model2.intercept ~== interceptExpected1 relTol 1E-3)
+ assert(model2.coefficients ~= coefficientsExpected1 relTol 1E-3)
+
+ // Bound constrained optimization with bound on both side.
+ val lowerBoundsOnCoefficients = Matrices.dense(1, 4, Array(0.0, -1.0, 0.0, -1.0))
+ val lowerBoundsOnIntercepts = Vectors.dense(0.0)
+
+ val trainer3 = new LogisticRegression()
+ .setUpperBoundsOnCoefficients(upperBoundsOnCoefficients)
+ .setUpperBoundsOnIntercepts(upperBoundsOnIntercepts)
+ .setLowerBoundsOnCoefficients(lowerBoundsOnCoefficients)
+ .setLowerBoundsOnIntercepts(lowerBoundsOnIntercepts)
+ .setFitIntercept(true)
+ .setStandardization(true)
+ .setWeightCol("weight")
+ val trainer4 = new LogisticRegression()
+ .setUpperBoundsOnCoefficients(upperBoundsOnCoefficients)
+ .setUpperBoundsOnIntercepts(upperBoundsOnIntercepts)
+ .setLowerBoundsOnCoefficients(lowerBoundsOnCoefficients)
+ .setLowerBoundsOnIntercepts(lowerBoundsOnIntercepts)
+ .setFitIntercept(true)
+ .setStandardization(false)
+ .setWeightCol("weight")
+
+ val model3 = trainer3.fit(binaryDataset)
+ val model4 = trainer4.fit(binaryDataset)
+
+ // The solution is generated by https://github.com/yanboliang/bound-optimization.
+ val coefficientsExpected3 = Vectors.dense(0.0, 0.0, 0.0, -0.71708632)
+ val interceptExpected3 = 0.58776113
+
+ assert(model3.intercept ~== interceptExpected3 relTol 1E-3)
+ assert(model3.coefficients ~= coefficientsExpected3 relTol 1E-3)
+
+ // Without regularization, with or without standardization will converge to the same solution.
+ assert(model4.intercept ~== interceptExpected3 relTol 1E-3)
+ assert(model4.coefficients ~= coefficientsExpected3 relTol 1E-3)
+
+ // Bound constrained optimization with infinite bound on both side.
+ val trainer5 = new LogisticRegression()
+ .setUpperBoundsOnCoefficients(Matrices.dense(1, 4, Array.fill(4)(Double.PositiveInfinity)))
+ .setUpperBoundsOnIntercepts(Vectors.dense(Double.PositiveInfinity))
+ .setLowerBoundsOnCoefficients(Matrices.dense(1, 4, Array.fill(4)(Double.NegativeInfinity)))
+ .setLowerBoundsOnIntercepts(Vectors.dense(Double.NegativeInfinity))
+ .setFitIntercept(true)
+ .setStandardization(true)
+ .setWeightCol("weight")
+ val trainer6 = new LogisticRegression()
+ .setUpperBoundsOnCoefficients(Matrices.dense(1, 4, Array.fill(4)(Double.PositiveInfinity)))
+ .setUpperBoundsOnIntercepts(Vectors.dense(Double.PositiveInfinity))
+ .setLowerBoundsOnCoefficients(Matrices.dense(1, 4, Array.fill(4)(Double.NegativeInfinity)))
+ .setLowerBoundsOnIntercepts(Vectors.dense(Double.NegativeInfinity))
+ .setFitIntercept(true)
+ .setStandardization(false)
+ .setWeightCol("weight")
+
+ val model5 = trainer5.fit(binaryDataset)
+ val model6 = trainer6.fit(binaryDataset)
+
+ // The solution is generated by https://github.com/yanboliang/bound-optimization.
+ // It should be same as unbound constrained optimization with LBFGS.
+ val coefficientsExpected5 = Vectors.dense(-0.5734389, 0.8911736, -0.3878645, -0.8060570)
+ val interceptExpected5 = 2.7355261
+
+ assert(model5.intercept ~== interceptExpected5 relTol 1E-3)
+ assert(model5.coefficients ~= coefficientsExpected5 relTol 1E-3)
+
+ // Without regularization, with or without standardization will converge to the same solution.
+ assert(model6.intercept ~== interceptExpected5 relTol 1E-3)
+ assert(model6.coefficients ~= coefficientsExpected5 relTol 1E-3)
+ }
+
test("binary logistic regression without intercept without regularization") {
val trainer1 = (new LogisticRegression).setFitIntercept(false).setStandardization(true)
.setWeightCol("weight")
@@ -650,6 +799,34 @@ class LogisticRegressionSuite
assert(model2.coefficients ~= coefficientsR relTol 1E-2)
}
+ test("binary logistic regression without intercept without regularization with bound") {
+ val upperBoundsOnCoefficients = Matrices.dense(1, 4, Array(1.0, 0.0, 1.0, 0.0)).toSparse
+
+ val trainer1 = new LogisticRegression()
+ .setUpperBoundsOnCoefficients(upperBoundsOnCoefficients)
+ .setFitIntercept(false)
+ .setStandardization(true)
+ .setWeightCol("weight")
+ val trainer2 = new LogisticRegression()
+ .setUpperBoundsOnCoefficients(upperBoundsOnCoefficients)
+ .setFitIntercept(false)
+ .setStandardization(false)
+ .setWeightCol("weight")
+
+ val model1 = trainer1.fit(binaryDataset)
+ val model2 = trainer2.fit(binaryDataset)
+
+ // The solution is generated by https://github.com/yanboliang/bound-optimization.
+ val coefficientsExpected = Vectors.dense(0.20847553, 0.0, -0.24240289, -0.55568071)
+
+ assert(model1.intercept ~== 0.0 relTol 1E-3)
+ assert(model1.coefficients ~= coefficientsExpected relTol 1E-3)
+
+ // Without regularization, with or without standardization will converge to the same solution.
+ assert(model2.intercept ~== 0.0 relTol 1E-3)
+ assert(model2.coefficients ~= coefficientsExpected relTol 1E-3)
+ }
+
test("binary logistic regression with intercept with L1 regularization") {
val trainer1 = (new LogisticRegression).setFitIntercept(true)
.setElasticNetParam(1.0).setRegParam(0.12).setStandardization(true).setWeightCol("weight")
@@ -815,6 +992,40 @@ class LogisticRegressionSuite
assert(model2.coefficients ~= coefficientsR relTol 1E-3)
}
+ test("binary logistic regression with intercept with L2 regularization with bound") {
+ val upperBoundsOnCoefficients = Matrices.dense(1, 4, Array(1.0, 0.0, 1.0, 0.0))
+ val upperBoundsOnIntercepts = Vectors.dense(1.0)
+
+ val trainer1 = new LogisticRegression()
+ .setUpperBoundsOnCoefficients(upperBoundsOnCoefficients)
+ .setUpperBoundsOnIntercepts(upperBoundsOnIntercepts)
+ .setRegParam(1.37)
+ .setFitIntercept(true)
+ .setStandardization(true)
+ .setWeightCol("weight")
+ val trainer2 = new LogisticRegression()
+ .setUpperBoundsOnCoefficients(upperBoundsOnCoefficients)
+ .setUpperBoundsOnIntercepts(upperBoundsOnIntercepts)
+ .setRegParam(1.37)
+ .setFitIntercept(true)
+ .setStandardization(false)
+ .setWeightCol("weight")
+
+ val model1 = trainer1.fit(binaryDataset)
+ val model2 = trainer2.fit(binaryDataset)
+
+ // The solution is generated by https://github.com/yanboliang/bound-optimization.
+ val coefficientsExpectedWithStd = Vectors.dense(-0.06985003, 0.0, -0.04794278, -0.10168595)
+ val interceptExpectedWithStd = 0.45750141
+ val coefficientsExpected = Vectors.dense(-0.0494524, 0.0, -0.11360797, -0.06313577)
+ val interceptExpected = 0.53722967
+
+ assert(model1.intercept ~== interceptExpectedWithStd relTol 1E-3)
+ assert(model1.coefficients ~= coefficientsExpectedWithStd relTol 1E-3)
+ assert(model2.intercept ~== interceptExpected relTol 1E-3)
+ assert(model2.coefficients ~= coefficientsExpected relTol 1E-3)
+ }
+
test("binary logistic regression without intercept with L2 regularization") {
val trainer1 = (new LogisticRegression).setFitIntercept(false)
.setElasticNetParam(0.0).setRegParam(1.37).setStandardization(true).setWeightCol("weight")
@@ -864,6 +1075,35 @@ class LogisticRegressionSuite
assert(model2.coefficients ~= coefficientsR relTol 1E-2)
}
+ test("binary logistic regression without intercept with L2 regularization with bound") {
+ val upperBoundsOnCoefficients = Matrices.dense(1, 4, Array(1.0, 0.0, 1.0, 0.0))
+
+ val trainer1 = new LogisticRegression()
+ .setUpperBoundsOnCoefficients(upperBoundsOnCoefficients)
+ .setRegParam(1.37)
+ .setFitIntercept(false)
+ .setStandardization(true)
+ .setWeightCol("weight")
+ val trainer2 = new LogisticRegression()
+ .setUpperBoundsOnCoefficients(upperBoundsOnCoefficients)
+ .setRegParam(1.37)
+ .setFitIntercept(false)
+ .setStandardization(false)
+ .setWeightCol("weight")
+
+ val model1 = trainer1.fit(binaryDataset)
+ val model2 = trainer2.fit(binaryDataset)
+
+ // The solution is generated by https://github.com/yanboliang/bound-optimization.
+ val coefficientsExpectedWithStd = Vectors.dense(-0.00796538, 0.0, -0.0394228, -0.0873314)
+ val coefficientsExpected = Vectors.dense(0.01105972, 0.0, -0.08574949, -0.05079558)
+
+ assert(model1.intercept ~== 0.0 relTol 1E-3)
+ assert(model1.coefficients ~= coefficientsExpectedWithStd relTol 1E-3)
+ assert(model2.intercept ~== 0.0 relTol 1E-3)
+ assert(model2.coefficients ~= coefficientsExpected relTol 1E-3)
+ }
+
test("binary logistic regression with intercept with ElasticNet regularization") {
val trainer1 = (new LogisticRegression).setFitIntercept(true).setMaxIter(200)
.setElasticNetParam(0.38).setRegParam(0.21).setStandardization(true).setWeightCol("weight")
@@ -1084,7 +1324,6 @@ class LogisticRegressionSuite
}
test("multinomial logistic regression with intercept without regularization") {
-
val trainer1 = (new LogisticRegression).setFitIntercept(true)
.setElasticNetParam(0.0).setRegParam(0.0).setStandardization(true).setWeightCol("weight")
val trainer2 = (new LogisticRegression).setFitIntercept(true)
@@ -1152,6 +1391,110 @@ class LogisticRegressionSuite
assert(model2.interceptVector.toArray.sum ~== 0.0 absTol eps)
}
+ test("multinomial logistic regression with intercept without regularization with bound") {
+ // Bound constrained optimization with bound on one side.
+ val lowerBoundsOnCoefficients = Matrices.dense(3, 4, Array.fill(12)(1.0))
+ val lowerBoundsOnIntercepts = Vectors.dense(Array.fill(3)(1.0))
+
+ val trainer1 = new LogisticRegression()
+ .setLowerBoundsOnCoefficients(lowerBoundsOnCoefficients)
+ .setLowerBoundsOnIntercepts(lowerBoundsOnIntercepts)
+ .setFitIntercept(true)
+ .setStandardization(true)
+ .setWeightCol("weight")
+ val trainer2 = new LogisticRegression()
+ .setLowerBoundsOnCoefficients(lowerBoundsOnCoefficients)
+ .setLowerBoundsOnIntercepts(lowerBoundsOnIntercepts)
+ .setFitIntercept(true)
+ .setStandardization(false)
+ .setWeightCol("weight")
+
+ val model1 = trainer1.fit(multinomialDataset)
+ val model2 = trainer2.fit(multinomialDataset)
+
+ // The solution is generated by https://github.com/yanboliang/bound-optimization.
+ val coefficientsExpected1 = new DenseMatrix(3, 4, Array(
+ 2.52076464, 2.73596057, 1.87984904, 2.73264492,
+ 1.93302281, 3.71363303, 1.50681746, 1.93398782,
+ 2.37839917, 1.93601818, 1.81924758, 2.45191255), isTransposed = true)
+ val interceptsExpected1 = Vectors.dense(1.00010477, 3.44237083, 4.86740286)
+
+ checkCoefficientsEquivalent(model1.coefficientMatrix, coefficientsExpected1)
+ assert(model1.interceptVector ~== interceptsExpected1 relTol 0.01)
+ checkCoefficientsEquivalent(model2.coefficientMatrix, coefficientsExpected1)
+ assert(model2.interceptVector ~== interceptsExpected1 relTol 0.01)
+
+ // Bound constrained optimization with bound on both side.
+ val upperBoundsOnCoefficients = Matrices.dense(3, 4, Array.fill(12)(2.0))
+ val upperBoundsOnIntercepts = Vectors.dense(Array.fill(3)(2.0))
+
+ val trainer3 = new LogisticRegression()
+ .setLowerBoundsOnCoefficients(lowerBoundsOnCoefficients)
+ .setLowerBoundsOnIntercepts(lowerBoundsOnIntercepts)
+ .setUpperBoundsOnCoefficients(upperBoundsOnCoefficients)
+ .setUpperBoundsOnIntercepts(upperBoundsOnIntercepts)
+ .setFitIntercept(true)
+ .setStandardization(true)
+ .setWeightCol("weight")
+ val trainer4 = new LogisticRegression()
+ .setLowerBoundsOnCoefficients(lowerBoundsOnCoefficients)
+ .setLowerBoundsOnIntercepts(lowerBoundsOnIntercepts)
+ .setUpperBoundsOnCoefficients(upperBoundsOnCoefficients)
+ .setUpperBoundsOnIntercepts(upperBoundsOnIntercepts)
+ .setFitIntercept(true)
+ .setStandardization(false)
+ .setWeightCol("weight")
+
+ val model3 = trainer3.fit(multinomialDataset)
+ val model4 = trainer4.fit(multinomialDataset)
+
+ // The solution is generated by https://github.com/yanboliang/bound-optimization.
+ val coefficientsExpected3 = new DenseMatrix(3, 4, Array(
+ 1.61967097, 1.16027835, 1.45131448, 1.97390431,
+ 1.30529317, 2.0, 1.12985473, 1.26652854,
+ 1.61647195, 1.0, 1.40642959, 1.72985589), isTransposed = true)
+ val interceptsExpected3 = Vectors.dense(1.0, 2.0, 2.0)
+
+ checkCoefficientsEquivalent(model3.coefficientMatrix, coefficientsExpected3)
+ assert(model3.interceptVector ~== interceptsExpected3 relTol 0.01)
+ checkCoefficientsEquivalent(model4.coefficientMatrix, coefficientsExpected3)
+ assert(model4.interceptVector ~== interceptsExpected3 relTol 0.01)
+
+ // Bound constrained optimization with infinite bound on both side.
+ val trainer5 = new LogisticRegression()
+ .setLowerBoundsOnCoefficients(Matrices.dense(3, 4, Array.fill(12)(Double.NegativeInfinity)))
+ .setLowerBoundsOnIntercepts(Vectors.dense(Array.fill(3)(Double.NegativeInfinity)))
+ .setUpperBoundsOnCoefficients(Matrices.dense(3, 4, Array.fill(12)(Double.PositiveInfinity)))
+ .setUpperBoundsOnIntercepts(Vectors.dense(Array.fill(3)(Double.PositiveInfinity)))
+ .setFitIntercept(true)
+ .setStandardization(true)
+ .setWeightCol("weight")
+ val trainer6 = new LogisticRegression()
+ .setLowerBoundsOnCoefficients(Matrices.dense(3, 4, Array.fill(12)(Double.NegativeInfinity)))
+ .setLowerBoundsOnIntercepts(Vectors.dense(Array.fill(3)(Double.NegativeInfinity)))
+ .setUpperBoundsOnCoefficients(Matrices.dense(3, 4, Array.fill(12)(Double.PositiveInfinity)))
+ .setUpperBoundsOnIntercepts(Vectors.dense(Array.fill(3)(Double.PositiveInfinity)))
+ .setFitIntercept(true)
+ .setStandardization(false)
+ .setWeightCol("weight")
+
+ val model5 = trainer5.fit(multinomialDataset)
+ val model6 = trainer6.fit(multinomialDataset)
+
+ // The solution is generated by https://github.com/yanboliang/bound-optimization.
+ // It should be same as unbound constrained optimization with LBFGS.
+ val coefficientsExpected5 = new DenseMatrix(3, 4, Array(
+ 0.24337896, -0.05916156, 0.14446790, 0.35976165,
+ -0.3443375, 0.9181331, -0.2283959, -0.4388066,
+ 0.10095851, -0.85897154, 0.08392798, 0.07904499), isTransposed = true)
+ val interceptsExpected5 = Vectors.dense(-2.10320093, 0.3394473, 1.76375361)
+
+ checkCoefficientsEquivalent(model5.coefficientMatrix, coefficientsExpected5)
+ assert(model5.interceptVector ~== interceptsExpected5 relTol 0.01)
+ checkCoefficientsEquivalent(model6.coefficientMatrix, coefficientsExpected5)
+ assert(model6.interceptVector ~== interceptsExpected5 relTol 0.01)
+ }
+
test("multinomial logistic regression without intercept without regularization") {
val trainer1 = (new LogisticRegression).setFitIntercept(false)
@@ -1220,6 +1563,35 @@ class LogisticRegressionSuite
assert(model2.interceptVector.toArray.sum ~== 0.0 absTol eps)
}
+ test("multinomial logistic regression without intercept without regularization with bound") {
+ val lowerBoundsOnCoefficients = Matrices.dense(3, 4, Array.fill(12)(1.0))
+
+ val trainer1 = new LogisticRegression()
+ .setLowerBoundsOnCoefficients(lowerBoundsOnCoefficients)
+ .setFitIntercept(false)
+ .setStandardization(true)
+ .setWeightCol("weight")
+ val trainer2 = new LogisticRegression()
+ .setLowerBoundsOnCoefficients(lowerBoundsOnCoefficients)
+ .setFitIntercept(false)
+ .setStandardization(false)
+ .setWeightCol("weight")
+
+ val model1 = trainer1.fit(multinomialDataset)
+ val model2 = trainer2.fit(multinomialDataset)
+
+ // The solution is generated by https://github.com/yanboliang/bound-optimization.
+ val coefficientsExpected = new DenseMatrix(3, 4, Array(
+ 1.62410051, 1.38219391, 1.34486618, 1.74641729,
+ 1.23058989, 2.71787825, 1.0, 1.00007073,
+ 1.79478632, 1.14360459, 1.33011603, 1.55093897), isTransposed = true)
+
+ checkCoefficientsEquivalent(model1.coefficientMatrix, coefficientsExpected)
+ assert(model1.interceptVector.toArray === Array.fill(3)(0.0))
+ checkCoefficientsEquivalent(model2.coefficientMatrix, coefficientsExpected)
+ assert(model2.interceptVector.toArray === Array.fill(3)(0.0))
+ }
+
test("multinomial logistic regression with intercept with L1 regularization") {
// use tighter constraints because OWL-QN solver takes longer to converge
@@ -1518,6 +1890,46 @@ class LogisticRegressionSuite
assert(model2.interceptVector.toArray.sum ~== 0.0 absTol eps)
}
+ test("multinomial logistic regression with intercept with L2 regularization with bound") {
+ val lowerBoundsOnCoefficients = Matrices.dense(3, 4, Array.fill(12)(1.0))
+ val lowerBoundsOnIntercepts = Vectors.dense(Array.fill(3)(1.0))
+
+ val trainer1 = new LogisticRegression()
+ .setLowerBoundsOnCoefficients(lowerBoundsOnCoefficients)
+ .setLowerBoundsOnIntercepts(lowerBoundsOnIntercepts)
+ .setRegParam(0.1)
+ .setFitIntercept(true)
+ .setStandardization(true)
+ .setWeightCol("weight")
+ val trainer2 = new LogisticRegression()
+ .setLowerBoundsOnCoefficients(lowerBoundsOnCoefficients)
+ .setLowerBoundsOnIntercepts(lowerBoundsOnIntercepts)
+ .setRegParam(0.1)
+ .setFitIntercept(true)
+ .setStandardization(false)
+ .setWeightCol("weight")
+
+ val model1 = trainer1.fit(multinomialDataset)
+ val model2 = trainer2.fit(multinomialDataset)
+
+ // The solution is generated by https://github.com/yanboliang/bound-optimization.
+ val coefficientsExpectedWithStd = new DenseMatrix(3, 4, Array(
+ 1.0, 1.0, 1.0, 1.01647497,
+ 1.0, 1.44105616, 1.0, 1.0,
+ 1.0, 1.0, 1.0, 1.0), isTransposed = true)
+ val interceptsExpectedWithStd = Vectors.dense(2.52055893, 1.0, 2.560682)
+ val coefficientsExpected = new DenseMatrix(3, 4, Array(
+ 1.0, 1.0, 1.03189386, 1.0,
+ 1.0, 1.0, 1.0, 1.0,
+ 1.0, 1.0, 1.0, 1.0), isTransposed = true)
+ val interceptsExpected = Vectors.dense(1.06418835, 1.0, 1.20494701)
+
+ assert(model1.coefficientMatrix ~== coefficientsExpectedWithStd relTol 0.01)
+ assert(model1.interceptVector ~== interceptsExpectedWithStd relTol 0.01)
+ assert(model2.coefficientMatrix ~== coefficientsExpected relTol 0.01)
+ assert(model2.interceptVector ~== interceptsExpected relTol 0.01)
+ }
+
test("multinomial logistic regression without intercept with L2 regularization") {
val trainer1 = (new LogisticRegression).setFitIntercept(false)
.setElasticNetParam(0.0).setRegParam(0.1).setStandardization(true).setWeightCol("weight")
@@ -1615,6 +2027,41 @@ class LogisticRegressionSuite
assert(model2.interceptVector.toArray.sum ~== 0.0 absTol eps)
}
+ test("multinomial logistic regression without intercept with L2 regularization with bound") {
+ val lowerBoundsOnCoefficients = Matrices.dense(3, 4, Array.fill(12)(1.0))
+
+ val trainer1 = new LogisticRegression()
+ .setLowerBoundsOnCoefficients(lowerBoundsOnCoefficients)
+ .setRegParam(0.1)
+ .setFitIntercept(false)
+ .setStandardization(true)
+ .setWeightCol("weight")
+ val trainer2 = new LogisticRegression()
+ .setLowerBoundsOnCoefficients(lowerBoundsOnCoefficients)
+ .setRegParam(0.1)
+ .setFitIntercept(false)
+ .setStandardization(false)
+ .setWeightCol("weight")
+
+ val model1 = trainer1.fit(multinomialDataset)
+ val model2 = trainer2.fit(multinomialDataset)
+
+ // The solution is generated by https://github.com/yanboliang/bound-optimization.
+ val coefficientsExpectedWithStd = new DenseMatrix(3, 4, Array(
+ 1.01324653, 1.0, 1.0, 1.0415767,
+ 1.0, 1.0, 1.0, 1.0,
+ 1.02244888, 1.0, 1.0, 1.0), isTransposed = true)
+ val coefficientsExpected = new DenseMatrix(3, 4, Array(
+ 1.0, 1.0, 1.03932259, 1.0,
+ 1.0, 1.0, 1.0, 1.0,
+ 1.0, 1.0, 1.03274649, 1.0), isTransposed = true)
+
+ assert(model1.coefficientMatrix ~== coefficientsExpectedWithStd absTol 0.01)
+ assert(model1.interceptVector.toArray === Array.fill(3)(0.0))
+ assert(model2.coefficientMatrix ~== coefficientsExpected absTol 0.01)
+ assert(model2.interceptVector.toArray === Array.fill(3)(0.0))
+ }
+
test("multinomial logistic regression with intercept with elasticnet regularization") {
val trainer1 = (new LogisticRegression).setFitIntercept(true).setWeightCol("weight")
.setElasticNetParam(0.5).setRegParam(0.1).setStandardization(true)
@@ -2273,4 +2720,19 @@ object LogisticRegressionSuite {
val testData = (0 until nPoints).map(i => LabeledPoint(y(i), x(i)))
testData
}
+
+ /**
+ * When no regularization is applied, the multinomial coefficients lack identifiability
+ * because we do not use a pivot class. We can add any constant value to the coefficients
+ * and get the same likelihood. If fitting under bound constrained optimization, we don't
+ * choose the mean centered coefficients like what we do for unbound problems, since they
+ * may out of the bounds. We use this function to check whether two coefficients are equivalent.
+ */
+ def checkCoefficientsEquivalent(coefficients1: Matrix, coefficients2: Matrix): Unit = {
+ coefficients1.colIter.zip(coefficients2.colIter).foreach { case (col1: Vector, col2: Vector) =>
+ (col1.asBreeze - col2.asBreeze).toArray.toSeq.sliding(2).foreach {
+ case Seq(v1, v2) => assert(v1 ~= v2 absTol 1E-3)
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org