You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by pa...@apache.org on 2014/07/04 22:33:34 UTC
[1/2] git commit: MAHOUT-1541, MAHOUT-1568,
MAHOUT-1569 fixed a build test problem,
drivers have an option new to not search for MAHOUT_HOME and SPARK_HOME
Repository: mahout
Updated Branches:
refs/heads/master 638ac8c7f -> 8b2bec7f5
MAHOUT-1541, MAHOUT-1568, MAHOUT-1569 fixed a build test problem, drivers have an option new to not search for MAHOUT_HOME and SPARK_HOME
Project: http://git-wip-us.apache.org/repos/asf/mahout/repo
Commit: http://git-wip-us.apache.org/repos/asf/mahout/commit/32badb1d
Tree: http://git-wip-us.apache.org/repos/asf/mahout/tree/32badb1d
Diff: http://git-wip-us.apache.org/repos/asf/mahout/diff/32badb1d
Branch: refs/heads/master
Commit: 32badb1d360ddf514e6b253f2dea9ae7e5df078a
Parents: 2b65475
Author: pferrel <pa...@occamsmachete.com>
Authored: Fri Jul 4 11:23:20 2014 -0700
Committer: pferrel <pa...@occamsmachete.com>
Committed: Fri Jul 4 11:23:20 2014 -0700
----------------------------------------------------------------------
.../apache/mahout/drivers/ItemSimilarityDriver.scala | 12 +++++++++---
.../org/apache/mahout/drivers/MahoutDriver.scala | 4 ++--
.../mahout/drivers/ItemSimilarityDriverSuite.scala | 15 ++++++++++-----
3 files changed, 21 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mahout/blob/32badb1d/spark/src/main/scala/org/apache/mahout/drivers/ItemSimilarityDriver.scala
----------------------------------------------------------------------
diff --git a/spark/src/main/scala/org/apache/mahout/drivers/ItemSimilarityDriver.scala b/spark/src/main/scala/org/apache/mahout/drivers/ItemSimilarityDriver.scala
index 77005f1..f78c590 100644
--- a/spark/src/main/scala/org/apache/mahout/drivers/ItemSimilarityDriver.scala
+++ b/spark/src/main/scala/org/apache/mahout/drivers/ItemSimilarityDriver.scala
@@ -167,6 +167,11 @@ object ItemSimilarityDriver extends MahoutDriver {
note("\nDefault delimiters will produce output of the form: \"itemID1<tab>itemID2:value2,itemID10:value10...\"")
+ //Jar inclusion, this option can be set when executing the driver from compiled code
+ opt[Unit]("dontAddMahoutJars") hidden() action { (_, options) =>
+ options.copy(dontAddMahoutJars = true) //set the value MahoutDriver so the context will be created correctly
+ }//Hidden option, used when executing tests or calling from other code where classes are all loaded explicitly
+
//Driver notes--driver specific
note("\nNote: Only the Log Likelihood Ratio (LLR) is supported as a similarity measure.\n")
@@ -196,14 +201,14 @@ object ItemSimilarityDriver extends MahoutDriver {
}
override def start(masterUrl: String = options.master,
- appName: String = options.appName):
+ appName: String = options.appName, dontAddMahoutJars: Boolean = options.dontAddMahoutJars):
Unit = {
sparkConf.set("spark.kryo.referenceTracking", "false")
.set("spark.kryoserializer.buffer.mb", "200")
.set("spark.executor.memory", options.sparkExecutorMem)
- super.start(masterUrl, appName)
+ super.start(masterUrl, appName, dontAddMahoutJars)
val readSchema1 = new Schema("delim" -> options.inDelim, "filter" -> options.filter1,
"rowIDPosition" -> options.rowIDPosition,
@@ -309,6 +314,7 @@ object ItemSimilarityDriver extends MahoutDriver {
inDelim: String = "[,\t ]",
rowKeyDelim: String = "\t",
columnIdStrengthDelim: String = ":",
- tupleDelim: String = ",")
+ tupleDelim: String = ",",
+ dontAddMahoutJars: Boolean = false)
}
http://git-wip-us.apache.org/repos/asf/mahout/blob/32badb1d/spark/src/main/scala/org/apache/mahout/drivers/MahoutDriver.scala
----------------------------------------------------------------------
diff --git a/spark/src/main/scala/org/apache/mahout/drivers/MahoutDriver.scala b/spark/src/main/scala/org/apache/mahout/drivers/MahoutDriver.scala
index afc7c1e..0c579d4 100644
--- a/spark/src/main/scala/org/apache/mahout/drivers/MahoutDriver.scala
+++ b/spark/src/main/scala/org/apache/mahout/drivers/MahoutDriver.scala
@@ -65,9 +65,9 @@ abstract class MahoutDriver {
mc = mahoutSparkContext(masterUrl, appName, customJars, sparkConf)
}
- protected def start(masterUrl: String, appName: String) : Unit = {
+ protected def start(masterUrl: String, appName: String, dontAddMahoutJars: Boolean = false) : Unit = {
val customJars = Traversable.empty[String]
- mc = mahoutSparkContext(masterUrl, appName, customJars, sparkConf)
+ mc = mahoutSparkContext(masterUrl, appName, customJars, sparkConf, !dontAddMahoutJars)
}
/** Override (optionally) for special cleanup */
http://git-wip-us.apache.org/repos/asf/mahout/blob/32badb1d/spark/src/test/scala/org/apache/mahout/drivers/ItemSimilarityDriverSuite.scala
----------------------------------------------------------------------
diff --git a/spark/src/test/scala/org/apache/mahout/drivers/ItemSimilarityDriverSuite.scala b/spark/src/test/scala/org/apache/mahout/drivers/ItemSimilarityDriverSuite.scala
index f649d7b..9f928e2 100644
--- a/spark/src/test/scala/org/apache/mahout/drivers/ItemSimilarityDriverSuite.scala
+++ b/spark/src/test/scala/org/apache/mahout/drivers/ItemSimilarityDriverSuite.scala
@@ -131,7 +131,8 @@ class ItemSimilarityDriverSuite extends FunSuite with MahoutSuite with MahoutLoc
"--inDelim", ",",
"--itemIDPosition", "2",
"--rowIDPosition", "0",
- "--filterPosition", "1"))
+ "--filterPosition", "1",
+ "--dontAddMahoutJars"))
beforeEach // restart the test context to read the output of the driver
val indicatorLines = mahoutCtx.textFile(OutPath+"/indicator-matrix/").collect.toSet[String]
@@ -185,7 +186,8 @@ class ItemSimilarityDriverSuite extends FunSuite with MahoutSuite with MahoutLoc
"--inDelim", "[,\t]",
"--itemIDPosition", "2",
"--rowIDPosition", "0",
- "--filterPosition", "1"))
+ "--filterPosition", "1",
+ "--dontAddMahoutJars"))
beforeEach // restart the test context to read the output of the driver
val indicatorLines = mahoutCtx.textFile(OutPath+"/indicator-matrix/").collect.toSet[String]
@@ -238,7 +240,8 @@ class ItemSimilarityDriverSuite extends FunSuite with MahoutSuite with MahoutLoc
"--inDelim", "\t",
"--itemIDPosition", "4",
"--rowIDPosition", "1",
- "--filterPosition", "2"))
+ "--filterPosition", "2",
+ "--dontAddMahoutJars"))
beforeEach // restart the test context to read the output of the driver
val indicatorLines = mahoutCtx.textFile(OutPath+"/indicator-matrix/").collect.toSet[String]
@@ -286,7 +289,8 @@ class ItemSimilarityDriverSuite extends FunSuite with MahoutSuite with MahoutLoc
ItemSimilarityDriver.main(Array(
"--input", InPath,
"--output", OutPath,
- "--master", masterUrl))
+ "--master", masterUrl,
+ "--dontAddMahoutJars"))
beforeEach // restart the test context to read the output of the driver
val indicatorLines = mahoutCtx.textFile(OutPath+"/indicator-matrix/").collect.toSet[String]
@@ -362,7 +366,8 @@ class ItemSimilarityDriverSuite extends FunSuite with MahoutSuite with MahoutLoc
"--rowIDPosition", "0",
"--filterPosition", "1",
"--filenamePattern", "m..tsv",
- "--recursive"))
+ "--recursive",
+ "--dontAddMahoutJars"))
beforeEach()// restart the test context to read the output of the driver
val indicatorLines = mahoutCtx.textFile(OutPath + "/indicator-matrix/").collect.toSet[String]
[2/2] git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/mahout
Posted by pa...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/mahout
Project: http://git-wip-us.apache.org/repos/asf/mahout/repo
Commit: http://git-wip-us.apache.org/repos/asf/mahout/commit/8b2bec7f
Tree: http://git-wip-us.apache.org/repos/asf/mahout/tree/8b2bec7f
Diff: http://git-wip-us.apache.org/repos/asf/mahout/diff/8b2bec7f
Branch: refs/heads/master
Commit: 8b2bec7f5e3a0d3500e38bbecbc3db6d08e7435c
Parents: 32badb1 638ac8c
Author: pferrel <pa...@occamsmachete.com>
Authored: Fri Jul 4 13:32:07 2014 -0700
Committer: pferrel <pa...@occamsmachete.com>
Committed: Fri Jul 4 13:32:07 2014 -0700
----------------------------------------------------------------------
.../apache/mahout/drivers/ItemSimilarityDriverSuite.scala | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mahout/blob/8b2bec7f/spark/src/test/scala/org/apache/mahout/drivers/ItemSimilarityDriverSuite.scala
----------------------------------------------------------------------