You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sz...@apache.org on 2015/04/23 04:34:50 UTC
[41/44] hive git commit: HIVE-10291 : Hive on Spark job configuration
needs to be logged [Spark Branch] (Szehon, reviewed by Chengxiang and Xuefu)
HIVE-10291 : Hive on Spark job configuration needs to be logged [Spark Branch] (Szehon, reviewed by Chengxiang and Xuefu)
git-svn-id: https://svn.apache.org/repos/asf/hive/branches/spark@1672733 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/31bec328
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/31bec328
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/31bec328
Branch: refs/heads/master
Commit: 31bec3286cf3dd22b1e9c1cea000db10ab9b517d
Parents: 1ac630c
Author: Szehon Ho <sz...@apache.org>
Authored: Fri Apr 10 19:34:19 2015 +0000
Committer: Szehon Ho <sz...@cloudera.com>
Committed: Wed Apr 22 19:33:53 2015 -0700
----------------------------------------------------------------------
.../hive/ql/exec/spark/RemoteHiveSparkClient.java | 17 ++++++++++++++++-
1 file changed, 16 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/31bec328/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
index 6701a29..059016d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java
@@ -22,6 +22,7 @@ import com.google.common.base.Strings;
import java.io.IOException;
import java.io.Serializable;
+import java.io.StringWriter;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
@@ -33,6 +34,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -231,6 +233,7 @@ public class RemoteHiveSparkClient implements HiveSparkClient {
Path localScratchDir = KryoSerializer.deserialize(scratchDirBytes, Path.class);
SparkWork localSparkWork = KryoSerializer.deserialize(sparkWorkBytes, SparkWork.class);
+ logConfigurations(localJobConf);
SparkCounters sparkCounters = new SparkCounters(jc.sc());
Map<String, List<String>> prefixes = localSparkWork.getRequiredCounterPrefix();
@@ -255,6 +258,18 @@ public class RemoteHiveSparkClient implements HiveSparkClient {
jc.monitor(future, sparkCounters, plan.getCachedRDDIds());
return null;
}
- }
+ private void logConfigurations(JobConf localJobConf) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Logging job configuration: ");
+ StringWriter outWriter = new StringWriter();
+ try {
+ Configuration.dumpConfiguration(localJobConf, outWriter);
+ } catch (IOException e) {
+ LOG.warn("Error logging job configuration", e);
+ }
+ LOG.info(outWriter.toString());
+ }
+ }
+ }
}