You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2016/02/10 19:17:48 UTC
svn commit: r1729685 - in /pig/trunk: ./
src/org/apache/pig/backend/hadoop/executionengine/tez/
src/org/apache/pig/backend/hadoop/executionengine/tez/plan/
Author: rohini
Date: Wed Feb 10 18:17:48 2016
New Revision: 1729685
URL: http://svn.apache.org/viewvc?rev=1729685&view=rev
Log:
PIG-4800: EvalFunc.getCacheFiles() fails for different namenode (rohini)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobCompiler.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezResourceManager.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezOperPlan.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1729685&r1=1729684&r2=1729685&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Feb 10 18:17:48 2016
@@ -89,6 +89,8 @@ PIG-4639: Add better parser for Apache H
BUG FIXES
+PIG-4800: EvalFunc.getCacheFiles() fails for different namenode (rohini)
+
PIG-4790: Join after union fail due to UnionOptimizer (rohini)
PIG-4686: Backend code should not call AvroStorageUtils.getPaths (mitdesai via rohini)
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobCompiler.java?rev=1729685&r1=1729684&r2=1729685&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobCompiler.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobCompiler.java Wed Feb 10 18:17:48 2016
@@ -100,10 +100,7 @@ public class TezJobCompiler {
}
String cacheFiles = pigContext.getProperties().getProperty("pig.streaming.cache.files");
if (cacheFiles != null) {
- for (String file : cacheFiles.split(",")) {
- // Do new URI() before passing to Path constructor else it encodes # when there is symlink
- TezResourceManager.getInstance().addTezResource(new Path(new URI(file.trim())).toUri());
- }
+ addCacheResources(cacheFiles.split(","));
}
for (Map.Entry<String, LocalResource> entry : localResources.entrySet()) {
log.info("Local resource: " + entry.getKey());
@@ -119,6 +116,20 @@ public class TezJobCompiler {
}
}
+ private void addCacheResources(String[] fileNames) throws Exception {
+ for (String fileName : fileNames) {
+ fileName = fileName.trim();
+ if (fileName.length() > 0) {
+ URI resourceURI = new URI(fileName);
+ String fragment = resourceURI.getFragment();
+
+ Path remoteFsPath = new Path(resourceURI);
+ String resourceName = (fragment != null && fragment.length() > 0) ? fragment : remoteFsPath.getName();
+ TezResourceManager.getInstance().addTezResource(resourceName, remoteFsPath);
+ }
+ }
+ }
+
private String createDagInfo(String script) throws IOException {
String dagInfo;
try {
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezResourceManager.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezResourceManager.java?rev=1729685&r1=1729684&r2=1729685&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezResourceManager.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezResourceManager.java Wed Feb 10 18:17:48 2016
@@ -130,7 +130,8 @@ public class TezResourceManager {
// The resource name will be symlinked to the resource path in the
// container's working directory.
Path resourcePath = resources.get(resourceName);
- FileStatus fstat = remoteFs.getFileStatus(resourcePath);
+ FileSystem fileSystem = resourcePath.getFileSystem(conf);
+ FileStatus fstat = fileSystem.getFileStatus(resourcePath);
LocalResource tezResource = LocalResource.newInstance(
ConverterUtils.getYarnUrlFromPath(fstat.getPath()),
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezOperPlan.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezOperPlan.java?rev=1729685&r1=1729684&r2=1729685&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezOperPlan.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezOperPlan.java Wed Feb 10 18:17:48 2016
@@ -154,7 +154,7 @@ public class TezOperPlan extends Operato
URI resourceURI = new URI(fileName);
String fragment = resourceURI.getFragment();
- Path remoteFsPath = new Path(resourceURI.getPath());
+ Path remoteFsPath = new Path(resourceURI);
String resourceName = (fragment != null && fragment.length() > 0) ? fragment : remoteFsPath.getName();
addExtraResource(resourceName, remoteFsPath);