You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by dl...@apache.org on 2014/08/16 03:27:15 UTC
git commit: (NOJIRA) temporary hack to enable (clamp down) on the
shell console logging.
Repository: mahout
Updated Branches:
refs/heads/master c6ee8cbcd -> 85a098a36
(NOJIRA) temporary hack to enable (clamp down) on the shell console logging.
Project: http://git-wip-us.apache.org/repos/asf/mahout/repo
Commit: http://git-wip-us.apache.org/repos/asf/mahout/commit/85a098a3
Tree: http://git-wip-us.apache.org/repos/asf/mahout/tree/85a098a3
Diff: http://git-wip-us.apache.org/repos/asf/mahout/diff/85a098a3
Branch: refs/heads/master
Commit: 85a098a364d467616394cd76d186d44f5f49a227
Parents: c6ee8cb
Author: Dmitriy Lyubimov <dl...@apache.org>
Authored: Fri Aug 15 18:25:43 2014 -0700
Committer: Dmitriy Lyubimov <dl...@apache.org>
Committed: Fri Aug 15 18:25:43 2014 -0700
----------------------------------------------------------------------
bin/mahout | 2 +-
conf/log4j.properties | 11 +++++++++++
.../mahout/sparkbindings/shell/MahoutSparkILoop.scala | 2 ++
.../org/apache/mahout/sparkbindings/shell/Main.scala | 8 ++++++++
.../scala/org/apache/mahout/sparkbindings/package.scala | 12 +++++++-----
5 files changed, 29 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mahout/blob/85a098a3/bin/mahout
----------------------------------------------------------------------
diff --git a/bin/mahout b/bin/mahout
index fafd5c3..d0623f1 100755
--- a/bin/mahout
+++ b/bin/mahout
@@ -212,7 +212,7 @@ else
fi
# add development dependencies to CLASSPATH
-if [ "$SPARK" == "" ]; then
+if [ "$SPARK" != "1" ]; then
for f in $MAHOUT_HOME/examples/target/dependency/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
http://git-wip-us.apache.org/repos/asf/mahout/blob/85a098a3/conf/log4j.properties
----------------------------------------------------------------------
diff --git a/conf/log4j.properties b/conf/log4j.properties
new file mode 100644
index 0000000..99a7d16
--- /dev/null
+++ b/conf/log4j.properties
@@ -0,0 +1,11 @@
+# Set everything to be logged to the console
+log4j.rootCategory=WARN, console
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
+
+# Settings to quiet third party logs that are too verbose
+log4j.logger.org.eclipse.jetty=WARN
+log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=WARN
+log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=WARN
http://git-wip-us.apache.org/repos/asf/mahout/blob/85a098a3/spark-shell/src/main/scala/org/apache/mahout/sparkbindings/shell/MahoutSparkILoop.scala
----------------------------------------------------------------------
diff --git a/spark-shell/src/main/scala/org/apache/mahout/sparkbindings/shell/MahoutSparkILoop.scala b/spark-shell/src/main/scala/org/apache/mahout/sparkbindings/shell/MahoutSparkILoop.scala
index b6b0851..0df42a3 100644
--- a/spark-shell/src/main/scala/org/apache/mahout/sparkbindings/shell/MahoutSparkILoop.scala
+++ b/spark-shell/src/main/scala/org/apache/mahout/sparkbindings/shell/MahoutSparkILoop.scala
@@ -8,6 +8,8 @@ import org.apache.mahout.sparkbindings._
class MahoutSparkILoop extends SparkILoop {
+ log.info("Mahout spark shell waking up.")
+
private val postInitScript =
"import org.apache.mahout.math._" ::
"import scalabindings._" ::
http://git-wip-us.apache.org/repos/asf/mahout/blob/85a098a3/spark-shell/src/main/scala/org/apache/mahout/sparkbindings/shell/Main.scala
----------------------------------------------------------------------
diff --git a/spark-shell/src/main/scala/org/apache/mahout/sparkbindings/shell/Main.scala b/spark-shell/src/main/scala/org/apache/mahout/sparkbindings/shell/Main.scala
index 9c0a51f..b59c9a7 100644
--- a/spark-shell/src/main/scala/org/apache/mahout/sparkbindings/shell/Main.scala
+++ b/spark-shell/src/main/scala/org/apache/mahout/sparkbindings/shell/Main.scala
@@ -17,12 +17,20 @@
package org.apache.mahout.sparkbindings.shell
+import org.apache.mahout.sparkbindings._
+import org.apache.log4j.PropertyConfigurator
+
object Main {
private var _interp: MahoutSparkILoop = _
def main(args:Array[String]) {
+
+ // Hack: for some very unclear reason, log4j is not picking up log4j.properties in Spark conf/ even
+ // though the latter is added to the classpath. So we force it to pick it.
+ PropertyConfigurator.configure(getMahoutHome() + "/conf/log4j.properties")
+
System.setProperty("scala.usejavacp", "true")
_interp = new MahoutSparkILoop()
// It looks like we need to initialize this too, since some Spark shell initilaization code
http://git-wip-us.apache.org/repos/asf/mahout/blob/85a098a3/spark/src/main/scala/org/apache/mahout/sparkbindings/package.scala
----------------------------------------------------------------------
diff --git a/spark/src/main/scala/org/apache/mahout/sparkbindings/package.scala b/spark/src/main/scala/org/apache/mahout/sparkbindings/package.scala
index 311cf82..6ca2a14 100644
--- a/spark/src/main/scala/org/apache/mahout/sparkbindings/package.scala
+++ b/spark/src/main/scala/org/apache/mahout/sparkbindings/package.scala
@@ -154,17 +154,19 @@ package object sparkbindings {
_canHaveMissingRows = canHaveMissingRows
)
- /** Acquire proper Mahout jars to be added to task context based on current MAHOUT_HOME. */
- private[sparkbindings] def findMahoutContextJars(closeables:java.util.Deque[Closeable]) = {
+ private[sparkbindings] def getMahoutHome() = {
var mhome = System.getenv("MAHOUT_HOME")
if (mhome == null) mhome = System.getProperty("mahout.home")
+ require(mhome != null, "MAHOUT_HOME is required to spawn mahout-based spark jobs" )
+ mhome
+ }
- if (mhome == null)
- throw new IllegalArgumentException("MAHOUT_HOME is required to spawn mahout-based spark jobs.")
+ /** Acquire proper Mahout jars to be added to task context based on current MAHOUT_HOME. */
+ private[sparkbindings] def findMahoutContextJars(closeables:java.util.Deque[Closeable]) = {
// Figure Mahout classpath using $MAHOUT_HOME/mahout classpath command.
- val fmhome = new File(mhome)
+ val fmhome = new File(getMahoutHome())
val bin = new File(fmhome, "bin")
val exec = new File(bin, "mahout")
if (!exec.canExecute)