You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by jk...@apache.org on 2017/03/21 18:01:30 UTC
spark git commit: [SPARK-20039][ML] rename ChiSquare to ChiSquareTest
Repository: spark
Updated Branches:
refs/heads/master 4c0ff5f58 -> ae4b91d1f
[SPARK-20039][ML] rename ChiSquare to ChiSquareTest
## What changes were proposed in this pull request?
I realized that since ChiSquare is in the package stat, it's pretty unclear if it's the hypothesis test, distribution, or what. This PR renames it to ChiSquareTest to clarify this.
## How was this patch tested?
Existing unit tests
Author: Joseph K. Bradley <jo...@databricks.com>
Closes #17368 from jkbradley/SPARK-20039.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ae4b91d1
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ae4b91d1
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ae4b91d1
Branch: refs/heads/master
Commit: ae4b91d1f5734b9d66f3b851b71b3c179f3cdd76
Parents: 4c0ff5f
Author: Joseph K. Bradley <jo...@databricks.com>
Authored: Tue Mar 21 11:01:25 2017 -0700
Committer: Joseph K. Bradley <jo...@databricks.com>
Committed: Tue Mar 21 11:01:25 2017 -0700
----------------------------------------------------------------------
.../org/apache/spark/ml/stat/ChiSquare.scala | 81 ----------------
.../apache/spark/ml/stat/ChiSquareTest.scala | 81 ++++++++++++++++
.../apache/spark/ml/stat/ChiSquareSuite.scala | 98 --------------------
.../spark/ml/stat/ChiSquareTestSuite.scala | 98 ++++++++++++++++++++
4 files changed, 179 insertions(+), 179 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/ae4b91d1/mllib/src/main/scala/org/apache/spark/ml/stat/ChiSquare.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/stat/ChiSquare.scala b/mllib/src/main/scala/org/apache/spark/ml/stat/ChiSquare.scala
deleted file mode 100644
index c3865ce..0000000
--- a/mllib/src/main/scala/org/apache/spark/ml/stat/ChiSquare.scala
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.ml.stat
-
-import org.apache.spark.annotation.{Experimental, Since}
-import org.apache.spark.ml.linalg.{Vector, Vectors, VectorUDT}
-import org.apache.spark.ml.util.SchemaUtils
-import org.apache.spark.mllib.linalg.{Vectors => OldVectors}
-import org.apache.spark.mllib.regression.{LabeledPoint => OldLabeledPoint}
-import org.apache.spark.mllib.stat.{Statistics => OldStatistics}
-import org.apache.spark.sql.DataFrame
-import org.apache.spark.sql.functions.col
-
-
-/**
- * :: Experimental ::
- *
- * Chi-square hypothesis testing for categorical data.
- *
- * See <a href="http://en.wikipedia.org/wiki/Chi-squared_test">Wikipedia</a> for more information
- * on the Chi-squared test.
- */
-@Experimental
-@Since("2.2.0")
-object ChiSquare {
-
- /** Used to construct output schema of tests */
- private case class ChiSquareResult(
- pValues: Vector,
- degreesOfFreedom: Array[Int],
- statistics: Vector)
-
- /**
- * Conduct Pearson's independence test for every feature against the label across the input RDD.
- * For each feature, the (feature, label) pairs are converted into a contingency matrix for which
- * the Chi-squared statistic is computed. All label and feature values must be categorical.
- *
- * The null hypothesis is that the occurrence of the outcomes is statistically independent.
- *
- * @param dataset DataFrame of categorical labels and categorical features.
- * Real-valued features will be treated as categorical for each distinct value.
- * @param featuresCol Name of features column in dataset, of type `Vector` (`VectorUDT`)
- * @param labelCol Name of label column in dataset, of any numerical type
- * @return DataFrame containing the test result for every feature against the label.
- * This DataFrame will contain a single Row with the following fields:
- * - `pValues: Vector`
- * - `degreesOfFreedom: Array[Int]`
- * - `statistics: Vector`
- * Each of these fields has one value per feature.
- */
- @Since("2.2.0")
- def test(dataset: DataFrame, featuresCol: String, labelCol: String): DataFrame = {
- val spark = dataset.sparkSession
- import spark.implicits._
-
- SchemaUtils.checkColumnType(dataset.schema, featuresCol, new VectorUDT)
- SchemaUtils.checkNumericType(dataset.schema, labelCol)
- val rdd = dataset.select(col(labelCol).cast("double"), col(featuresCol)).as[(Double, Vector)]
- .rdd.map { case (label, features) => OldLabeledPoint(label, OldVectors.fromML(features)) }
- val testResults = OldStatistics.chiSqTest(rdd)
- val pValues: Vector = Vectors.dense(testResults.map(_.pValue))
- val degreesOfFreedom: Array[Int] = testResults.map(_.degreesOfFreedom)
- val statistics: Vector = Vectors.dense(testResults.map(_.statistic))
- spark.createDataFrame(Seq(ChiSquareResult(pValues, degreesOfFreedom, statistics)))
- }
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/ae4b91d1/mllib/src/main/scala/org/apache/spark/ml/stat/ChiSquareTest.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/stat/ChiSquareTest.scala b/mllib/src/main/scala/org/apache/spark/ml/stat/ChiSquareTest.scala
new file mode 100644
index 0000000..21eba9a
--- /dev/null
+++ b/mllib/src/main/scala/org/apache/spark/ml/stat/ChiSquareTest.scala
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.stat
+
+import org.apache.spark.annotation.{Experimental, Since}
+import org.apache.spark.ml.linalg.{Vector, Vectors, VectorUDT}
+import org.apache.spark.ml.util.SchemaUtils
+import org.apache.spark.mllib.linalg.{Vectors => OldVectors}
+import org.apache.spark.mllib.regression.{LabeledPoint => OldLabeledPoint}
+import org.apache.spark.mllib.stat.{Statistics => OldStatistics}
+import org.apache.spark.sql.DataFrame
+import org.apache.spark.sql.functions.col
+
+
+/**
+ * :: Experimental ::
+ *
+ * Chi-square hypothesis testing for categorical data.
+ *
+ * See <a href="http://en.wikipedia.org/wiki/Chi-squared_test">Wikipedia</a> for more information
+ * on the Chi-squared test.
+ */
+@Experimental
+@Since("2.2.0")
+object ChiSquareTest {
+
+ /** Used to construct output schema of tests */
+ private case class ChiSquareResult(
+ pValues: Vector,
+ degreesOfFreedom: Array[Int],
+ statistics: Vector)
+
+ /**
+ * Conduct Pearson's independence test for every feature against the label across the input RDD.
+ * For each feature, the (feature, label) pairs are converted into a contingency matrix for which
+ * the Chi-squared statistic is computed. All label and feature values must be categorical.
+ *
+ * The null hypothesis is that the occurrence of the outcomes is statistically independent.
+ *
+ * @param dataset DataFrame of categorical labels and categorical features.
+ * Real-valued features will be treated as categorical for each distinct value.
+ * @param featuresCol Name of features column in dataset, of type `Vector` (`VectorUDT`)
+ * @param labelCol Name of label column in dataset, of any numerical type
+ * @return DataFrame containing the test result for every feature against the label.
+ * This DataFrame will contain a single Row with the following fields:
+ * - `pValues: Vector`
+ * - `degreesOfFreedom: Array[Int]`
+ * - `statistics: Vector`
+ * Each of these fields has one value per feature.
+ */
+ @Since("2.2.0")
+ def test(dataset: DataFrame, featuresCol: String, labelCol: String): DataFrame = {
+ val spark = dataset.sparkSession
+ import spark.implicits._
+
+ SchemaUtils.checkColumnType(dataset.schema, featuresCol, new VectorUDT)
+ SchemaUtils.checkNumericType(dataset.schema, labelCol)
+ val rdd = dataset.select(col(labelCol).cast("double"), col(featuresCol)).as[(Double, Vector)]
+ .rdd.map { case (label, features) => OldLabeledPoint(label, OldVectors.fromML(features)) }
+ val testResults = OldStatistics.chiSqTest(rdd)
+ val pValues: Vector = Vectors.dense(testResults.map(_.pValue))
+ val degreesOfFreedom: Array[Int] = testResults.map(_.degreesOfFreedom)
+ val statistics: Vector = Vectors.dense(testResults.map(_.statistic))
+ spark.createDataFrame(Seq(ChiSquareResult(pValues, degreesOfFreedom, statistics)))
+ }
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/ae4b91d1/mllib/src/test/scala/org/apache/spark/ml/stat/ChiSquareSuite.scala
----------------------------------------------------------------------
diff --git a/mllib/src/test/scala/org/apache/spark/ml/stat/ChiSquareSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/stat/ChiSquareSuite.scala
deleted file mode 100644
index b4bed82..0000000
--- a/mllib/src/test/scala/org/apache/spark/ml/stat/ChiSquareSuite.scala
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.ml.stat
-
-import java.util.Random
-
-import org.apache.spark.{SparkException, SparkFunSuite}
-import org.apache.spark.ml.feature.LabeledPoint
-import org.apache.spark.ml.linalg.{Vector, Vectors}
-import org.apache.spark.ml.util.DefaultReadWriteTest
-import org.apache.spark.ml.util.TestingUtils._
-import org.apache.spark.mllib.stat.test.ChiSqTest
-import org.apache.spark.mllib.util.MLlibTestSparkContext
-
-class ChiSquareSuite
- extends SparkFunSuite with MLlibTestSparkContext with DefaultReadWriteTest {
-
- import testImplicits._
-
- test("test DataFrame of labeled points") {
- // labels: 1.0 (2 / 6), 0.0 (4 / 6)
- // feature1: 0.5 (1 / 6), 1.5 (2 / 6), 3.5 (3 / 6)
- // feature2: 10.0 (1 / 6), 20.0 (1 / 6), 30.0 (2 / 6), 40.0 (2 / 6)
- val data = Seq(
- LabeledPoint(0.0, Vectors.dense(0.5, 10.0)),
- LabeledPoint(0.0, Vectors.dense(1.5, 20.0)),
- LabeledPoint(1.0, Vectors.dense(1.5, 30.0)),
- LabeledPoint(0.0, Vectors.dense(3.5, 30.0)),
- LabeledPoint(0.0, Vectors.dense(3.5, 40.0)),
- LabeledPoint(1.0, Vectors.dense(3.5, 40.0)))
- for (numParts <- List(2, 4, 6, 8)) {
- val df = spark.createDataFrame(sc.parallelize(data, numParts))
- val chi = ChiSquare.test(df, "features", "label")
- val (pValues: Vector, degreesOfFreedom: Array[Int], statistics: Vector) =
- chi.select("pValues", "degreesOfFreedom", "statistics")
- .as[(Vector, Array[Int], Vector)].head()
- assert(pValues ~== Vectors.dense(0.6873, 0.6823) relTol 1e-4)
- assert(degreesOfFreedom === Array(2, 3))
- assert(statistics ~== Vectors.dense(0.75, 1.5) relTol 1e-4)
- }
- }
-
- test("large number of features (SPARK-3087)") {
- // Test that the right number of results is returned
- val numCols = 1001
- val sparseData = Array(
- LabeledPoint(0.0, Vectors.sparse(numCols, Seq((100, 2.0)))),
- LabeledPoint(0.1, Vectors.sparse(numCols, Seq((200, 1.0)))))
- val df = spark.createDataFrame(sparseData)
- val chi = ChiSquare.test(df, "features", "label")
- val (pValues: Vector, degreesOfFreedom: Array[Int], statistics: Vector) =
- chi.select("pValues", "degreesOfFreedom", "statistics")
- .as[(Vector, Array[Int], Vector)].head()
- assert(pValues.size === numCols)
- assert(degreesOfFreedom.length === numCols)
- assert(statistics.size === numCols)
- assert(pValues(1000) !== null) // SPARK-3087
- }
-
- test("fail on continuous features or labels") {
- val tooManyCategories: Int = 100000
- assert(tooManyCategories > ChiSqTest.maxCategories, "This unit test requires that " +
- "tooManyCategories be large enough to cause ChiSqTest to throw an exception.")
-
- val random = new Random(11L)
- val continuousLabel = Seq.fill(tooManyCategories)(
- LabeledPoint(random.nextDouble(), Vectors.dense(random.nextInt(2))))
- withClue("ChiSquare should throw an exception when given a continuous-valued label") {
- intercept[SparkException] {
- val df = spark.createDataFrame(continuousLabel)
- ChiSquare.test(df, "features", "label")
- }
- }
- val continuousFeature = Seq.fill(tooManyCategories)(
- LabeledPoint(random.nextInt(2), Vectors.dense(random.nextDouble())))
- withClue("ChiSquare should throw an exception when given continuous-valued features") {
- intercept[SparkException] {
- val df = spark.createDataFrame(continuousFeature)
- ChiSquare.test(df, "features", "label")
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/ae4b91d1/mllib/src/test/scala/org/apache/spark/ml/stat/ChiSquareTestSuite.scala
----------------------------------------------------------------------
diff --git a/mllib/src/test/scala/org/apache/spark/ml/stat/ChiSquareTestSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/stat/ChiSquareTestSuite.scala
new file mode 100644
index 0000000..2d6aad0
--- /dev/null
+++ b/mllib/src/test/scala/org/apache/spark/ml/stat/ChiSquareTestSuite.scala
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.stat
+
+import java.util.Random
+
+import org.apache.spark.{SparkException, SparkFunSuite}
+import org.apache.spark.ml.feature.LabeledPoint
+import org.apache.spark.ml.linalg.{Vector, Vectors}
+import org.apache.spark.ml.util.DefaultReadWriteTest
+import org.apache.spark.ml.util.TestingUtils._
+import org.apache.spark.mllib.stat.test.ChiSqTest
+import org.apache.spark.mllib.util.MLlibTestSparkContext
+
+class ChiSquareTestSuite
+ extends SparkFunSuite with MLlibTestSparkContext with DefaultReadWriteTest {
+
+ import testImplicits._
+
+ test("test DataFrame of labeled points") {
+ // labels: 1.0 (2 / 6), 0.0 (4 / 6)
+ // feature1: 0.5 (1 / 6), 1.5 (2 / 6), 3.5 (3 / 6)
+ // feature2: 10.0 (1 / 6), 20.0 (1 / 6), 30.0 (2 / 6), 40.0 (2 / 6)
+ val data = Seq(
+ LabeledPoint(0.0, Vectors.dense(0.5, 10.0)),
+ LabeledPoint(0.0, Vectors.dense(1.5, 20.0)),
+ LabeledPoint(1.0, Vectors.dense(1.5, 30.0)),
+ LabeledPoint(0.0, Vectors.dense(3.5, 30.0)),
+ LabeledPoint(0.0, Vectors.dense(3.5, 40.0)),
+ LabeledPoint(1.0, Vectors.dense(3.5, 40.0)))
+ for (numParts <- List(2, 4, 6, 8)) {
+ val df = spark.createDataFrame(sc.parallelize(data, numParts))
+ val chi = ChiSquareTest.test(df, "features", "label")
+ val (pValues: Vector, degreesOfFreedom: Array[Int], statistics: Vector) =
+ chi.select("pValues", "degreesOfFreedom", "statistics")
+ .as[(Vector, Array[Int], Vector)].head()
+ assert(pValues ~== Vectors.dense(0.6873, 0.6823) relTol 1e-4)
+ assert(degreesOfFreedom === Array(2, 3))
+ assert(statistics ~== Vectors.dense(0.75, 1.5) relTol 1e-4)
+ }
+ }
+
+ test("large number of features (SPARK-3087)") {
+ // Test that the right number of results is returned
+ val numCols = 1001
+ val sparseData = Array(
+ LabeledPoint(0.0, Vectors.sparse(numCols, Seq((100, 2.0)))),
+ LabeledPoint(0.1, Vectors.sparse(numCols, Seq((200, 1.0)))))
+ val df = spark.createDataFrame(sparseData)
+ val chi = ChiSquareTest.test(df, "features", "label")
+ val (pValues: Vector, degreesOfFreedom: Array[Int], statistics: Vector) =
+ chi.select("pValues", "degreesOfFreedom", "statistics")
+ .as[(Vector, Array[Int], Vector)].head()
+ assert(pValues.size === numCols)
+ assert(degreesOfFreedom.length === numCols)
+ assert(statistics.size === numCols)
+ assert(pValues(1000) !== null) // SPARK-3087
+ }
+
+ test("fail on continuous features or labels") {
+ val tooManyCategories: Int = 100000
+ assert(tooManyCategories > ChiSqTest.maxCategories, "This unit test requires that " +
+ "tooManyCategories be large enough to cause ChiSqTest to throw an exception.")
+
+ val random = new Random(11L)
+ val continuousLabel = Seq.fill(tooManyCategories)(
+ LabeledPoint(random.nextDouble(), Vectors.dense(random.nextInt(2))))
+ withClue("ChiSquare should throw an exception when given a continuous-valued label") {
+ intercept[SparkException] {
+ val df = spark.createDataFrame(continuousLabel)
+ ChiSquareTest.test(df, "features", "label")
+ }
+ }
+ val continuousFeature = Seq.fill(tooManyCategories)(
+ LabeledPoint(random.nextInt(2), Vectors.dense(random.nextDouble())))
+ withClue("ChiSquare should throw an exception when given continuous-valued features") {
+ intercept[SparkException] {
+ val df = spark.createDataFrame(continuousFeature)
+ ChiSquareTest.test(df, "features", "label")
+ }
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org