You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2014/11/03 07:02:42 UTC
svn commit: r1636274 - in /pig/branches/branch-0.14: CHANGES.txt
src/org/apache/pig/backend/hadoop/executionengine/tez/TezResourceManager.java
Author: daijy
Date: Mon Nov 3 06:02:41 2014
New Revision: 1636274
URL: http://svn.apache.org/r1636274
Log:
PIG-4261: Skip shipping local resources in tez local mode
Modified:
pig/branches/branch-0.14/CHANGES.txt
pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/TezResourceManager.java
Modified: pig/branches/branch-0.14/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.14/CHANGES.txt?rev=1636274&r1=1636273&r2=1636274&view=diff
==============================================================================
--- pig/branches/branch-0.14/CHANGES.txt (original)
+++ pig/branches/branch-0.14/CHANGES.txt Mon Nov 3 06:02:41 2014
@@ -97,6 +97,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-4261: Skip shipping local resources in tez local mode (daijy)
+
PIG-4182: e2e tests Scripting_[1-12] fail on Windows (daijy)
PIG-4259: Fix few issues related to Union, CROSS and auto parallelism in Tez (rohini)
Modified: pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/TezResourceManager.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/TezResourceManager.java?rev=1636274&r1=1636273&r2=1636274&view=diff
==============================================================================
--- pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/TezResourceManager.java (original)
+++ pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/tez/TezResourceManager.java Mon Nov 3 06:02:41 2014
@@ -42,6 +42,7 @@ public class TezResourceManager {
private Path stagingDir;
private FileSystem remoteFs;
private Configuration conf;
+ private PigContext pigContext;
public Map<String, Path> resources = new HashMap<String, Path>();
static public TezResourceManager getInstance() {
@@ -56,6 +57,7 @@ public class TezResourceManager {
this.stagingDir = FileLocalizer.getTemporaryResourcePath(pigContext);
this.remoteFs = FileSystem.get(conf);
this.conf = conf;
+ this.pigContext = pigContext;
this.inited = true;
}
}
@@ -76,7 +78,7 @@ public class TezResourceManager {
}
// Ship the local resource to the staging directory on the remote FS
- if (uri.toString().startsWith("file:")) {
+ if (!pigContext.getExecType().isLocal() && uri.toString().startsWith("file:")) {
Path remoteFsPath = remoteFs.makeQualified(new Path(stagingDir, resourceName));
remoteFs.copyFromLocalFile(resourcePath, remoteFsPath);
remoteFs.setReplication(remoteFsPath, (short)conf.getInt(Job.SUBMIT_REPLICATION, 3));