You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2013/11/25 23:40:22 UTC
svn commit: r1545434 - in
/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec:
Utilities.java mr/ExecMapper.java mr/ExecReducer.java
tez/MapRecordProcessor.java tez/ObjectCache.java
tez/ReduceRecordProcessor.java
Author: gunther
Date: Mon Nov 25 22:40:22 2013
New Revision: 1545434
URL: http://svn.apache.org/r1545434
Log:
HIVE-5883: Plan is deserialized more often than necessary on Tez (in container reuse case) (Gunther Hagleitner)
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecReducer.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ObjectCache.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1545434&r1=1545433&r2=1545434&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Mon Nov 25 22:40:22 2013
@@ -265,15 +265,31 @@ public final class Utilities {
return w;
}
+ public static void setMapWork(Configuration conf, MapWork work) {
+ setBaseWork(conf, MAP_PLAN_NAME, work);
+ }
+
public static MapWork getMapWork(Configuration conf) {
return (MapWork) getBaseWork(conf, MAP_PLAN_NAME);
}
+ public static void setReduceWork(Configuration conf, ReduceWork work) {
+ setBaseWork(conf, REDUCE_PLAN_NAME, work);
+ }
+
public static ReduceWork getReduceWork(Configuration conf) {
return (ReduceWork) getBaseWork(conf, REDUCE_PLAN_NAME);
}
/**
+ * Pushes work into the global work map
+ */
+ public static void setBaseWork(Configuration conf, String name, BaseWork work) {
+ Path path = getPlanPath(conf, name);
+ gWorkMap.put(path, work);
+ }
+
+ /**
* Returns the Map or Reduce plan
* Side effect: the BaseWork returned is also placed in the gWorkMap
* @param conf
@@ -288,8 +304,7 @@ public final class Utilities {
try {
path = getPlanPath(conf, name);
assert path != null;
- gWork = gWorkMap.get(path);
- if (gWork == null) {
+ if (!gWorkMap.containsKey(path)) {
Path localPath;
if (ShimLoader.getHadoopShims().isLocalMode(conf)) {
localPath = path;
@@ -334,6 +349,7 @@ public final class Utilities {
gWorkMap.put(path, gWork);
} else {
LOG.debug("Found plan in cache.");
+ gWork = gWorkMap.get(path);
}
return gWork;
} catch (FileNotFoundException fnf) {
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java?rev=1545434&r1=1545433&r2=1545434&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java Mon Nov 25 22:40:22 2013
@@ -110,6 +110,8 @@ public class ExecMapper extends MapReduc
if (mrwork == null) {
mrwork = Utilities.getMapWork(job);
cache.cache(PLAN_KEY, mrwork);
+ } else {
+ Utilities.setMapWork(job, mrwork);
}
if (mrwork.getVectorMode()) {
mo = new VectorMapOperator();
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecReducer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecReducer.java?rev=1545434&r1=1545433&r2=1545434&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecReducer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecReducer.java Mon Nov 25 22:40:22 2013
@@ -122,6 +122,8 @@ public class ExecReducer extends MapRedu
if (gWork == null) {
gWork = Utilities.getReduceWork(job);
cache.cache(PLAN_KEY, gWork);
+ } else {
+ Utilities.setReduceWork(job, gWork);
}
reducer = gWork.getReducer();
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java?rev=1545434&r1=1545433&r2=1545434&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java Mon Nov 25 22:40:22 2013
@@ -95,6 +95,8 @@ public class MapRecordProcessor extends
for (String s: mapWork.getAliases()) {
l4j.info("Alias: "+s);
}
+ } else {
+ Utilities.setMapWork(jconf, mapWork);
}
if (mapWork.getVectorMode()) {
mapOp = new VectorMapOperator();
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ObjectCache.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ObjectCache.java?rev=1545434&r1=1545433&r2=1545434&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ObjectCache.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ObjectCache.java Mon Nov 25 22:40:22 2013
@@ -36,14 +36,16 @@ public class ObjectCache implements org.
@Override
public void cache(String key, Object value) {
- LOG.info("Adding " + key + " to cache");
+ LOG.info("Adding " + key + " to cache with value " + value);
registry.add(ObjectLifeCycle.VERTEX, key, value);
}
@Override
public Object retrieve(String key) {
Object o = registry.get(key);
- LOG.info("Found " + key + " in cache with value: " + o);
+ if (o != null) {
+ LOG.info("Found " + key + " in cache with value: " + o);
+ }
return o;
}
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java?rev=1545434&r1=1545433&r2=1545434&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java Mon Nov 25 22:40:22 2013
@@ -104,6 +104,8 @@ public class ReduceRecordProcessor exte
if (redWork == null) {
redWork = Utilities.getReduceWork(jconf);
cache.cache(REDUCE_PLAN_KEY, redWork);
+ } else {
+ Utilities.setReduceWork(jconf, redWork);
}
reducer = redWork.getReducer();