You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2015/07/15 23:12:56 UTC
svn commit: r1691274 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
src/org/apache/pig/backend/hadoop/executionengine/tez/TezResourceManager.java
Author: rohini
Date: Wed Jul 15 21:12:55 2015
New Revision: 1691274
URL: http://svn.apache.org/r1691274
Log:
PIG-4618: When use tez as the engine , set pig.user.cache.enabled=true do not take effect (wisgood via rohini)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezResourceManager.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1691274&r1=1691273&r2=1691274&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Jul 15 21:12:55 2015
@@ -32,6 +32,8 @@ PIG-4570: Allow AvroStorage to use a cla
BUG FIXES
+PIG-4618: When use tez as the engine , set pig.user.cache.enabled=true do not take effect (wisgood via rohini)
+
PIG-4533: Document error: Pig does support concatenated gz file (xhudik via daijy)
PIG-4578: ToDateISO should support optional ' ' space variant used by JDBC (michaelthoward via daijy)
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java?rev=1691274&r1=1691273&r2=1691274&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java Wed Jul 15 21:12:55 2015
@@ -1708,7 +1708,7 @@ public class JobControlCompiler{
return null;
}
- private static Path getCacheStagingDir(Configuration conf) throws IOException {
+ public static Path getCacheStagingDir(Configuration conf) throws IOException {
String pigTempDir = conf.get(PigConfiguration.PIG_USER_CACHE_LOCATION,
conf.get(PigConfiguration.PIG_TEMP_DIR, "/tmp"));
String currentUser = System.getProperty("user.name");
@@ -1719,7 +1719,7 @@ public class JobControlCompiler{
return stagingDir;
}
- private static Path getFromCache(PigContext pigContext,
+ public static Path getFromCache(PigContext pigContext,
Configuration conf,
URL url) throws IOException {
InputStream is1 = null;
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezResourceManager.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezResourceManager.java?rev=1691274&r1=1691273&r2=1691274&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezResourceManager.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezResourceManager.java Wed Jul 15 21:12:55 2015
@@ -35,6 +35,9 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.io.FileLocalizer;
+import org.apache.pig.PigConfiguration;
+import static org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobControlCompiler.getFromCache;
+import static org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobControlCompiler.getCacheStagingDir;
public class TezResourceManager {
private static TezResourceManager instance = null;
@@ -79,6 +82,18 @@ public class TezResourceManager {
// Ship the local resource to the staging directory on the remote FS
if (!pigContext.getExecType().isLocal() && uri.toString().startsWith("file:")) {
+ boolean cacheEnabled =
+ conf.getBoolean(PigConfiguration.PIG_USER_CACHE_ENABLED, false);
+
+ if(cacheEnabled){
+ Path pathOnDfs = getFromCache(pigContext, conf, uri.toURL());
+ if(pathOnDfs != null) {
+ resources.put(resourceName, pathOnDfs);
+ return pathOnDfs;
+ }
+
+ }
+
Path remoteFsPath = remoteFs.makeQualified(new Path(stagingDir, resourceName));
remoteFs.copyFromLocalFile(resourcePath, remoteFsPath);
remoteFs.setReplication(remoteFsPath, (short)conf.getInt(Job.SUBMIT_REPLICATION, 3));