You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2013/08/13 01:35:56 UTC
svn commit: r1513295 - in
/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez:
DagUtils.java TezTask.java
Author: gunther
Date: Mon Aug 12 23:35:56 2013
New Revision: 1513295
URL: http://svn.apache.org/r1513295
Log:
HIVE-5040: Yarn resource names cannot contain slashes (Gunther Hagleitner)
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java?rev=1513295&r1=1513294&r2=1513295&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java Mon Aug 12 23:35:56 2013
@@ -28,6 +28,7 @@ import java.util.Map;
import javax.security.auth.login.LoginException;
import org.apache.commons.lang.StringUtils;
+import org.apache.commons.io.FilenameUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -200,11 +201,11 @@ public class DagUtils {
if (localWorkLr != null) {
localResources.put(hashTableArchive.getName(), localWorkLr);
}
- localResources.put(appJarLr.getResource().getFile(), appJarLr);
+ localResources.put(getBaseName(appJarLr), appJarLr);
for (LocalResource lr: additionalLr) {
- localResources.put(lr.getResource().getFile(), lr);
+ localResources.put(getBaseName(lr), lr);
}
- localResources.put(planPath.getName(), planLr);
+ localResources.put(FilenameUtils.getName(planPath.getName()), planLr);
MRHelpers.updateLocalResourcesForInputSplits(FileSystem.get(conf), inputSplitInfo,
localResources);
@@ -269,11 +270,11 @@ public class DagUtils {
reducer.setJavaOpts(MRHelpers.getReduceJavaOpts(conf));
Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
- localResources.put(appJarLr.getResource().getFile(), appJarLr);
+ localResources.put(getBaseName(appJarLr), appJarLr);
for (LocalResource lr: additionalLr) {
- localResources.put(lr.getResource().getFile(), lr);
+ localResources.put(getBaseName(lr), lr);
}
- localResources.put(planPath.getName(), planLr);
+ localResources.put(FilenameUtils.getName(planPath.getName()), planLr);
reducer.setTaskLocalResources(localResources);
return reducer;
@@ -393,7 +394,13 @@ public class DagUtils {
// returns the location on disc of the jar of this class.
return DagUtils.class.getProtectionDomain().getCodeSource().getLocation().toURI().toString();
}
-
+
+ /*
+ * Helper function to retrieve the basename of a local resource
+ */
+ public static String getBaseName(LocalResource lr) {
+ return FilenameUtils.getName(lr.getResource().getFile());
+ }
/**
* @param pathStr - the string from which we try to determine the resource base name
@@ -546,7 +553,6 @@ public class DagUtils {
hiveConf.setBoolean("mapred.mapper.new-api", false);
JobConf conf = (JobConf) MRHelpers.getBaseMRConfiguration();
- MRHelpers.doJobClientMagic(conf);
for (Map.Entry<String, String> entry: hiveConf) {
if (conf.get(entry.getKey()) == null) {
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java?rev=1513295&r1=1513294&r2=1513295&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java Mon Aug 12 23:35:56 2013
@@ -153,7 +153,7 @@ public class TezTask extends Task<TezWor
// setup local resources used by application master
Map<String, LocalResource> amLrs = new HashMap<String, LocalResource>();
- amLrs.put(appJarLr.getResource().getFile(), appJarLr);
+ amLrs.put(DagUtils.getBaseName(appJarLr), appJarLr);
// ready to start execution on the cluster
DAGClient dagClient = tezClient.submitDAGApplication(dag, scratchDir,