You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2020/05/01 14:17:00 UTC
[spark] branch master updated: [SPARK-31307][ML][EXAMPLES] Add
examples for ml.fvalue
This is an automated email from the ASF dual-hosted git repository.
srowen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 348fd53 [SPARK-31307][ML][EXAMPLES] Add examples for ml.fvalue
348fd53 is described below
commit 348fd53214ccc476bee37e3ddd6b075a53886104
Author: Qianyang Yu <qy...@us.ibm.com>
AuthorDate: Fri May 1 09:16:08 2020 -0500
[SPARK-31307][ML][EXAMPLES] Add examples for ml.fvalue
### What changes were proposed in this pull request?
Add FValue example for ml.stat.FValueTest in python/java/scala
### Why are the changes needed?
Improve ML example
### Does this PR introduce any user-facing change?
No
### How was this patch tested?
manually run the example
Closes #28400 from kevinyu98/spark-26111-fvalue-examples.
Authored-by: Qianyang Yu <qy...@us.ibm.com>
Signed-off-by: Sean Owen <sr...@gmail.com>
---
.../spark/examples/ml/JavaFValueTestExample.java | 75 ++++++++++++++++++++++
examples/src/main/python/ml/fvalue_test_example.py | 52 +++++++++++++++
.../spark/examples/ml/FVlaueTestExample.scala | 63 ++++++++++++++++++
3 files changed, 190 insertions(+)
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaFValueTestExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaFValueTestExample.java
new file mode 100644
index 0000000..11861ac
--- /dev/null
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaFValueTestExample.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.examples.ml;
+
+import org.apache.spark.sql.SparkSession;
+
+// $example on$
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.spark.ml.linalg.Vectors;
+import org.apache.spark.ml.linalg.VectorUDT;
+import org.apache.spark.ml.stat.FValueTest;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.RowFactory;
+import org.apache.spark.sql.types.*;
+// $example off$
+
+/**
+ * An example for FValue testing.
+ * Run with
+ * <pre>
+ * bin/run-example ml.JavaFValueTestExample
+ * </pre>
+ */
+public class JavaFValueTestExample {
+
+ public static void main(String[] args) {
+ SparkSession spark = SparkSession
+ .builder()
+ .appName("JavaFValueTestExample")
+ .getOrCreate();
+
+ // $example on$
+ List<Row> data = Arrays.asList(
+ RowFactory.create(4.6, Vectors.dense(6.0, 7.0, 0.0, 7.0, 6.0, 0.0)),
+ RowFactory.create(6.6, Vectors.dense(0.0, 9.0, 6.0, 0.0, 5.0, 9.0)),
+ RowFactory.create(5.1, Vectors.dense(0.0, 9.0, 3.0, 0.0, 5.0, 5.0)),
+ RowFactory.create(7.6, Vectors.dense(0.0, 9.0, 8.0, 5.0, 6.0, 4.0)),
+ RowFactory.create(9.0, Vectors.dense(8.0, 9.0, 6.0, 5.0, 4.0, 4.0)),
+ RowFactory.create(9.0, Vectors.dense(8.0, 9.0, 6.0, 4.0, 0.0, 0.0))
+ );
+
+ StructType schema = new StructType(new StructField[]{
+ new StructField("label", DataTypes.DoubleType, false, Metadata.empty()),
+ new StructField("features", new VectorUDT(), false, Metadata.empty()),
+ });
+
+ Dataset<Row> df = spark.createDataFrame(data, schema);
+ Row r = FValueTest.test(df, "features", "label").head();
+ System.out.println("pValues: " + r.get(0).toString());
+ System.out.println("degreesOfFreedom: " + r.getList(1).toString());
+ System.out.println("fvalue: " + r.get(2).toString());
+
+ // $example off$
+
+ spark.stop();
+ }
+}
diff --git a/examples/src/main/python/ml/fvalue_test_example.py b/examples/src/main/python/ml/fvalue_test_example.py
new file mode 100644
index 0000000..4a97bcd
--- /dev/null
+++ b/examples/src/main/python/ml/fvalue_test_example.py
@@ -0,0 +1,52 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+An example for FValue testing.
+Run with:
+ bin/spark-submit examples/src/main/python/ml/fvalue_test_example.py
+"""
+from __future__ import print_function
+
+from pyspark.sql import SparkSession
+# $example on$
+from pyspark.ml.linalg import Vectors
+from pyspark.ml.stat import FValueTest
+# $example off$
+
+if __name__ == "__main__":
+ spark = SparkSession \
+ .builder \
+ .appName("FValueTestExample") \
+ .getOrCreate()
+
+ # $example on$
+ data = [(4.6, Vectors.dense(6.0, 7.0, 0.0, 7.0, 6.0, 0.0)),
+ (6.6, Vectors.dense(0.0, 9.0, 6.0, 0.0, 5.0, 9.0)),
+ (5.1, Vectors.dense(0.0, 9.0, 3.0, 0.0, 5.0, 5.0)),
+ (7.6, Vectors.dense(0.0, 9.0, 8.0, 5.0, 6.0, 4.0)),
+ (9.0, Vectors.dense(8.0, 9.0, 6.0, 5.0, 4.0, 4.0)),
+ (9.0, Vectors.dense(8.0, 9.0, 6.0, 4.0, 0.0, 0.0))]
+ df = spark.createDataFrame(data, ["label", "features"])
+
+ ftest = FValueTest.test(df, "features", "label").head()
+ print("pValues: " + str(ftest.pValues))
+ print("degreesOfFreedom: " + str(ftest.degreesOfFreedom))
+ print("fvalue: " + str(ftest.fValues))
+ # $example off$
+
+ spark.stop()
diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/FVlaueTestExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/FVlaueTestExample.scala
new file mode 100644
index 0000000..08ec22c
--- /dev/null
+++ b/examples/src/main/scala/org/apache/spark/examples/ml/FVlaueTestExample.scala
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// scalastyle:off println
+package org.apache.spark.examples.ml
+
+// $example on$
+import org.apache.spark.ml.linalg.{Vector, Vectors}
+import org.apache.spark.ml.stat.FValueTest
+// $example off$
+import org.apache.spark.sql.SparkSession
+
+/**
+ * An example for FValue testing.
+ * Run with
+ * {{{
+ * bin/run-example ml.FValueTestExample
+ * }}}
+ */
+object FValueTestExample {
+
+ def main(args: Array[String]): Unit = {
+ val spark = SparkSession
+ .builder
+ .appName("FValueTestExample")
+ .getOrCreate()
+ import spark.implicits._
+
+ // $example on$
+ val data = Seq(
+ (4.6, Vectors.dense(6.0, 7.0, 0.0, 7.0, 6.0, 0.0)),
+ (6.6, Vectors.dense(0.0, 9.0, 6.0, 0.0, 5.0, 9.0)),
+ (5.1, Vectors.dense(0.0, 9.0, 3.0, 0.0, 5.0, 5.0)),
+ (7.6, Vectors.dense(0.0, 9.0, 8.0, 5.0, 6.0, 4.0)),
+ (9.0, Vectors.dense(8.0, 9.0, 6.0, 5.0, 4.0, 4.0)),
+ (9.0, Vectors.dense(8.0, 9.0, 6.0, 4.0, 0.0, 0.0))
+ )
+
+ val df = data.toDF("label", "features")
+ val fValue = FValueTest.test(df, "features", "label").head
+ println(s"pValues ${fValue.getAs[Vector](0)}")
+ println(s"degreesOfFreedom ${fValue.getSeq[Int](1).mkString("[", ",", "]")}")
+ println(s"fValues ${fValue.getAs[Vector](2)}")
+ // $example off$
+
+ spark.stop()
+ }
+}
+// scalastyle:on println
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org