You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2015/02/03 22:08:47 UTC
svn commit: r1656921 - in /pig/trunk: CHANGES.txt conf/pig.properties
src/org/apache/pig/PigConfiguration.java
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
Author: rohini
Date: Tue Feb 3 21:08:46 2015
New Revision: 1656921
URL: http://svn.apache.org/r1656921
Log:
PIG-4407: Allow specifying a replication factor for jarcache (jira.shegalov via rohini)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/conf/pig.properties
pig/trunk/src/org/apache/pig/PigConfiguration.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1656921&r1=1656920&r2=1656921&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Feb 3 21:08:46 2015
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
IMPROVEMENTS
+PIG-4407: Allow specifying a replication factor for jarcache (jira.shegalov via rohini)
+
PIG-4401: Add pattern matching to PluckTuple (cheolsoo)
PIG-2692: Make the Pig unit faciliities more generalizable and update javadocs (razsapps via daijy)
Modified: pig/trunk/conf/pig.properties
URL: http://svn.apache.org/viewvc/pig/trunk/conf/pig.properties?rev=1656921&r1=1656920&r2=1656921&view=diff
==============================================================================
--- pig/trunk/conf/pig.properties (original)
+++ pig/trunk/conf/pig.properties Tue Feb 3 21:08:46 2015
@@ -150,6 +150,11 @@
#
# pig.user.cache.location=/tmp
+# Replication factor for cached jars. If not specified mapred.submit.replication
+# is used, whose default is 10.
+#
+# pig.user.cache.replication=10
+
# Default UTC offset. (default: the host's current UTC offset) Supply a UTC
# offset in Java's timezone format: e.g., +08:00.
#
Modified: pig/trunk/src/org/apache/pig/PigConfiguration.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/PigConfiguration.java?rev=1656921&r1=1656920&r2=1656921&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/PigConfiguration.java (original)
+++ pig/trunk/src/org/apache/pig/PigConfiguration.java Tue Feb 3 21:08:46 2015
@@ -268,6 +268,11 @@ public class PigConfiguration {
public static final String PIG_USER_CACHE_LOCATION = "pig.user.cache.location";
/**
+ * Replication factor for files in pig jar cache
+ */
+ public static final String PIG_USER_CACHE_REPLICATION = "pig.user.cache.replication";
+
+ /**
* Comma-delimited entries of commands/operators that must be disallowed.
* This is a security feature to be used by administrators to block use of
* commands by users. For eg, an admin might like to block all filesystem
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java?rev=1656921&r1=1656920&r2=1656921&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java Tue Feb 3 21:08:46 2015
@@ -1799,7 +1799,10 @@ public class JobControlCompiler{
// attempt to copy to cache else return null
fs.mkdirs(cacheDir, FileLocalizer.OWNER_ONLY_PERMS);
is2 = url.openStream();
- os = FileSystem.create(fs, cacheFile, FileLocalizer.OWNER_ONLY_PERMS);
+ short replication = (short)conf.getInt(PigConfiguration.PIG_USER_CACHE_REPLICATION,
+ conf.getInt("mapred.submit.replication", 10));
+ os = fs.create(cacheFile, replication);
+ fs.setPermission(cacheFile, FileLocalizer.OWNER_ONLY_PERMS);
IOUtils.copyBytes(is2, os, 4096, true);
return cacheFile;