You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by me...@apache.org on 2014/12/03 11:50:22 UTC
spark git commit: [SPARK-4710] [mllib] Eliminate MLlib compilation
warnings
Repository: spark
Updated Branches:
refs/heads/master 8af551f71 -> 4ac215115
[SPARK-4710] [mllib] Eliminate MLlib compilation warnings
Renamed StreamingKMeans to StreamingKMeansExample to avoid warning about name conflict with StreamingKMeans class.
Added import to DecisionTreeRunner to eliminate warning.
CC: mengxr
Author: Joseph K. Bradley <jo...@databricks.com>
Closes #3568 from jkbradley/ml-compilation-warnings and squashes the following commits:
64d6bc4 [Joseph K. Bradley] Updated DecisionTreeRunner.scala and StreamingKMeans.scala to eliminate compilation warnings, including renaming StreamingKMeans to StreamingKMeansExample.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/4ac21511
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/4ac21511
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/4ac21511
Branch: refs/heads/master
Commit: 4ac21511547dc6227d05bf61821cd2d9ab5ede74
Parents: 8af551f
Author: Joseph K. Bradley <jo...@databricks.com>
Authored: Wed Dec 3 18:50:03 2014 +0800
Committer: Xiangrui Meng <me...@databricks.com>
Committed: Wed Dec 3 18:50:03 2014 +0800
----------------------------------------------------------------------
.../examples/mllib/DecisionTreeRunner.scala | 2 +
.../spark/examples/mllib/StreamingKMeans.scala | 77 --------------------
.../examples/mllib/StreamingKMeansExample.scala | 77 ++++++++++++++++++++
3 files changed, 79 insertions(+), 77 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/4ac21511/examples/src/main/scala/org/apache/spark/examples/mllib/DecisionTreeRunner.scala
----------------------------------------------------------------------
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/DecisionTreeRunner.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/DecisionTreeRunner.scala
index 98f9d16..54953ad 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/DecisionTreeRunner.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/DecisionTreeRunner.scala
@@ -17,6 +17,8 @@
package org.apache.spark.examples.mllib
+import scala.language.reflectiveCalls
+
import scopt.OptionParser
import org.apache.spark.{SparkConf, SparkContext}
http://git-wip-us.apache.org/repos/asf/spark/blob/4ac21511/examples/src/main/scala/org/apache/spark/examples/mllib/StreamingKMeans.scala
----------------------------------------------------------------------
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/StreamingKMeans.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/StreamingKMeans.scala
deleted file mode 100644
index 33e5760..0000000
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/StreamingKMeans.scala
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.examples.mllib
-
-import org.apache.spark.mllib.linalg.Vectors
-import org.apache.spark.mllib.regression.LabeledPoint
-import org.apache.spark.mllib.clustering.StreamingKMeans
-import org.apache.spark.SparkConf
-import org.apache.spark.streaming.{Seconds, StreamingContext}
-
-/**
- * Estimate clusters on one stream of data and make predictions
- * on another stream, where the data streams arrive as text files
- * into two different directories.
- *
- * The rows of the training text files must be vector data in the form
- * `[x1,x2,x3,...,xn]`
- * Where n is the number of dimensions.
- *
- * The rows of the test text files must be labeled data in the form
- * `(y,[x1,x2,x3,...,xn])`
- * Where y is some identifier. n must be the same for train and test.
- *
- * Usage: StreamingKmeans <trainingDir> <testDir> <batchDuration> <numClusters> <numDimensions>
- *
- * To run on your local machine using the two directories `trainingDir` and `testDir`,
- * with updates every 5 seconds, 2 dimensions per data point, and 3 clusters, call:
- * $ bin/run-example \
- * org.apache.spark.examples.mllib.StreamingKMeans trainingDir testDir 5 3 2
- *
- * As you add text files to `trainingDir` the clusters will continuously update.
- * Anytime you add text files to `testDir`, you'll see predicted labels using the current model.
- *
- */
-object StreamingKMeans {
-
- def main(args: Array[String]) {
- if (args.length != 5) {
- System.err.println(
- "Usage: StreamingKMeans " +
- "<trainingDir> <testDir> <batchDuration> <numClusters> <numDimensions>")
- System.exit(1)
- }
-
- val conf = new SparkConf().setMaster("local").setAppName("StreamingLinearRegression")
- val ssc = new StreamingContext(conf, Seconds(args(2).toLong))
-
- val trainingData = ssc.textFileStream(args(0)).map(Vectors.parse)
- val testData = ssc.textFileStream(args(1)).map(LabeledPoint.parse)
-
- val model = new StreamingKMeans()
- .setK(args(3).toInt)
- .setDecayFactor(1.0)
- .setRandomCenters(args(4).toInt, 0.0)
-
- model.trainOn(trainingData)
- model.predictOnValues(testData.map(lp => (lp.label, lp.features))).print()
-
- ssc.start()
- ssc.awaitTermination()
- }
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/4ac21511/examples/src/main/scala/org/apache/spark/examples/mllib/StreamingKMeansExample.scala
----------------------------------------------------------------------
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/StreamingKMeansExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/StreamingKMeansExample.scala
new file mode 100644
index 0000000..8bb12d2
--- /dev/null
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/StreamingKMeansExample.scala
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.examples.mllib
+
+import org.apache.spark.SparkConf
+import org.apache.spark.mllib.clustering.StreamingKMeans
+import org.apache.spark.mllib.linalg.Vectors
+import org.apache.spark.mllib.regression.LabeledPoint
+import org.apache.spark.streaming.{Seconds, StreamingContext}
+
+/**
+ * Estimate clusters on one stream of data and make predictions
+ * on another stream, where the data streams arrive as text files
+ * into two different directories.
+ *
+ * The rows of the training text files must be vector data in the form
+ * `[x1,x2,x3,...,xn]`
+ * Where n is the number of dimensions.
+ *
+ * The rows of the test text files must be labeled data in the form
+ * `(y,[x1,x2,x3,...,xn])`
+ * Where y is some identifier. n must be the same for train and test.
+ *
+ * Usage:
+ * StreamingKMeansExample <trainingDir> <testDir> <batchDuration> <numClusters> <numDimensions>
+ *
+ * To run on your local machine using the two directories `trainingDir` and `testDir`,
+ * with updates every 5 seconds, 2 dimensions per data point, and 3 clusters, call:
+ * $ bin/run-example mllib.StreamingKMeansExample trainingDir testDir 5 3 2
+ *
+ * As you add text files to `trainingDir` the clusters will continuously update.
+ * Anytime you add text files to `testDir`, you'll see predicted labels using the current model.
+ *
+ */
+object StreamingKMeansExample {
+
+ def main(args: Array[String]) {
+ if (args.length != 5) {
+ System.err.println(
+ "Usage: StreamingKMeansExample " +
+ "<trainingDir> <testDir> <batchDuration> <numClusters> <numDimensions>")
+ System.exit(1)
+ }
+
+ val conf = new SparkConf().setMaster("local").setAppName("StreamingKMeansExample")
+ val ssc = new StreamingContext(conf, Seconds(args(2).toLong))
+
+ val trainingData = ssc.textFileStream(args(0)).map(Vectors.parse)
+ val testData = ssc.textFileStream(args(1)).map(LabeledPoint.parse)
+
+ val model = new StreamingKMeans()
+ .setK(args(3).toInt)
+ .setDecayFactor(1.0)
+ .setRandomCenters(args(4).toInt, 0.0)
+
+ model.trainOn(trainingData)
+ model.predictOnValues(testData.map(lp => (lp.label, lp.features))).print()
+
+ ssc.start()
+ ssc.awaitTermination()
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org