You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2013/11/25 23:32:34 UTC
svn commit: r1545429 - in
/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql:
exec/MapOperator.java exec/Utilities.java exec/tez/RecordProcessor.java
exec/vector/VectorReduceSinkOperator.java io/orc/OrcInputFormat.java
Author: gunther
Date: Mon Nov 25 22:32:33 2013
New Revision: 1545429
URL: http://svn.apache.org/r1545429
Log:
HIVE-5882: Reduce logging verbosity on Tez (Gunther Hagleitner)
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/RecordProcessor.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorReduceSinkOperator.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java?rev=1545429&r1=1545428&r2=1545429&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java Mon Nov 25 22:32:33 2013
@@ -350,8 +350,10 @@ public class MapOperator extends Operato
for (String onealias : aliases) {
Operator<? extends OperatorDesc> op = conf.getAliasToWork().get(onealias);
- LOG.info("Adding alias " + onealias + " to work list for file "
- + onefile);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Adding alias " + onealias + " to work list for file "
+ + onefile);
+ }
MapInputPath inp = new MapInputPath(onefile, onealias, op, partDesc);
if (opCtxMap.containsKey(inp)) {
continue;
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1545429&r1=1545428&r2=1545429&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Mon Nov 25 22:32:33 2013
@@ -298,10 +298,10 @@ public final class Utilities {
}
if (HiveConf.getBoolVar(conf, ConfVars.HIVE_RPC_QUERY_PLAN)) {
- LOG.info("Loading plan from: "+path.toUri().getPath());
+ LOG.debug("Loading plan from: "+path.toUri().getPath());
String planString = conf.get(path.toUri().getPath());
if (planString == null) {
- LOG.info("Could not find plan!");
+ LOG.debug("Could not find plan string in conf");
return null;
}
byte[] planBytes = Base64.decodeBase64(planString);
@@ -332,6 +332,8 @@ public final class Utilities {
}
}
gWorkMap.put(path, gWork);
+ } else {
+ LOG.debug("Found plan in cache.");
}
return gWork;
} catch (FileNotFoundException fnf) {
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/RecordProcessor.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/RecordProcessor.java?rev=1545429&r1=1545428&r2=1545429&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/RecordProcessor.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/RecordProcessor.java Mon Nov 25 22:32:33 2013
@@ -72,18 +72,19 @@ public abstract class RecordProcessor {
// Allocate the bean at the beginning -
memoryMXBean = ManagementFactory.getMemoryMXBean();
-
l4j.info("maximum memory = " + memoryMXBean.getHeapMemoryUsage().getMax());
isLogInfoEnabled = l4j.isInfoEnabled();
//log classpaths
try {
- l4j.info("conf classpath = "
- + Arrays.asList(((URLClassLoader) jconf.getClassLoader()).getURLs()));
- l4j.info("thread classpath = "
- + Arrays.asList(((URLClassLoader) Thread.currentThread()
- .getContextClassLoader()).getURLs()));
+ if (l4j.isDebugEnabled()) {
+ l4j.debug("conf classpath = "
+ + Arrays.asList(((URLClassLoader) jconf.getClassLoader()).getURLs()));
+ l4j.debug("thread classpath = "
+ + Arrays.asList(((URLClassLoader) Thread.currentThread()
+ .getContextClassLoader()).getURLs()));
+ }
} catch (Exception e) {
l4j.info("cannot get classpath: " + e.getMessage());
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorReduceSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorReduceSinkOperator.java?rev=1545429&r1=1545428&r2=1545429&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorReduceSinkOperator.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorReduceSinkOperator.java Mon Nov 25 22:32:33 2013
@@ -137,7 +137,7 @@ public class VectorReduceSinkOperator ex
colNames = String.format("%s %s", colNames, colName);
}
- LOG.info(String.format("keyObjectInspector [%s]%s => %s",
+ LOG.debug(String.format("keyObjectInspector [%s]%s => %s",
keyObjectInspector.getClass(),
keyObjectInspector,
colNames));
@@ -169,7 +169,7 @@ public class VectorReduceSinkOperator ex
colNames = String.format("%s %s", colNames, colName);
}
- LOG.info(String.format("valueObjectInspector [%s]%s => %s",
+ LOG.debug(String.format("valueObjectInspector [%s]%s => %s",
valueObjectInspector.getClass(),
valueObjectInspector,
colNames));
@@ -198,7 +198,7 @@ public class VectorReduceSinkOperator ex
public void processOp(Object row, int tag) throws HiveException {
VectorizedRowBatch vrg = (VectorizedRowBatch) row;
- LOG.info(String.format("sinking %d rows, %d values, %d keys, %d parts",
+ LOG.debug(String.format("sinking %d rows, %d values, %d keys, %d parts",
vrg.size,
valueEval.length,
keyEval.length,
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java?rev=1545429&r1=1545428&r2=1545429&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java Mon Nov 25 22:32:33 2013
@@ -185,7 +185,7 @@ public class OrcInputFormat implements
if (serializedPushdown == null
|| (conf.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR) == null
&& conf.get(serdeConstants.LIST_COLUMNS) == null)) {
- LOG.info("No ORC pushdown predicate");
+ LOG.debug("No ORC pushdown predicate");
return null;
}
SearchArgument sarg = SearchArgument.FACTORY.create
@@ -197,7 +197,9 @@ public class OrcInputFormat implements
public static String[] getIncludedColumnNames(
List<OrcProto.Type> types, boolean[] includedColumns, Configuration conf) {
String columnNamesString = conf.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR);
- LOG.info("included columns names = " + columnNamesString);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("included columns names = " + columnNamesString);
+ }
if (columnNamesString == null || conf.get(TableScanDesc.FILTER_EXPR_CONF_STR) == null) {
return null;
}
@@ -508,13 +510,14 @@ public class OrcInputFormat implements
} else {
// Invalidate
Context.footerCache.invalidate(file.getPath());
- LOG.info("Meta-Info for : " + file.getPath() + " changed. CachedModificationTime: "
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Meta-Info for : " + file.getPath() + " changed. CachedModificationTime: "
+ fileInfo.modificationTime + ", CurrentModificationTime: "
+ file.getModificationTime()
+ ", CachedLength: " + fileInfo.size + ", CurrentLength: " + file.getLen());
+ }
}
} else {
- LOG.info("Info not cached for path: " + file.getPath());
if (LOG.isDebugEnabled()) {
LOG.debug("Info not cached for path: " + file.getPath());
}