You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2014/01/27 09:01:59 UTC
svn commit: r1561598 - in /pig/branches/branch-0.12: ./
shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/
shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/
src/org/apache/pig/backend/hadoop/datastorage/
Author: rohini
Date: Mon Jan 27 08:01:58 2014
New Revision: 1561598
URL: http://svn.apache.org/r1561598
Log:
PIG-3677: ConfigurationUtil.getLocalFSProperties can return an inconsistent property set (rohini)
Modified:
pig/branches/branch-0.12/CHANGES.txt
pig/branches/branch-0.12/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java
pig/branches/branch-0.12/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java
pig/branches/branch-0.12/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java
Modified: pig/branches/branch-0.12/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.12/CHANGES.txt?rev=1561598&r1=1561597&r2=1561598&view=diff
==============================================================================
--- pig/branches/branch-0.12/CHANGES.txt (original)
+++ pig/branches/branch-0.12/CHANGES.txt Mon Jan 27 08:01:58 2014
@@ -32,6 +32,8 @@ PIG-3480: TFile-based tmpfile compressio
BUG FIXES
+PIG-3677: ConfigurationUtil.getLocalFSProperties can return an inconsistent property set (rohini)
+
PIG-3621: Python Avro library can't read Avros made with builtin AvroStorage (rusell.jurney via cheolsoo)
PIG-3592: Should not try to create success file for non-fs schemes like hbase (rohini)
Modified: pig/branches/branch-0.12/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.12/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java?rev=1561598&r1=1561597&r2=1561598&view=diff
==============================================================================
--- pig/branches/branch-0.12/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java (original)
+++ pig/branches/branch-0.12/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java Mon Jan 27 08:01:58 2014
@@ -97,7 +97,7 @@ public class HadoopShims {
public static JobControl newJobControl(String groupName, int timeToSleep) {
return new PigJobControl(groupName, timeToSleep);
}
-
+
public static long getDefaultBlockSize(FileSystem fs, Path path) {
return fs.getDefaultBlockSize();
}
@@ -115,4 +115,8 @@ public class HadoopShims {
// for Hadoop 0.20
return report.getProgress() != successfulProgress;
}
+
+ public static void unsetConf(Configuration conf, String key) {
+ // Not supported in Hadoop 0.20/1.x
+ }
}
Modified: pig/branches/branch-0.12/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.12/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java?rev=1561598&r1=1561597&r2=1561598&view=diff
==============================================================================
--- pig/branches/branch-0.12/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java (original)
+++ pig/branches/branch-0.12/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java Mon Jan 27 08:01:58 2014
@@ -115,4 +115,8 @@ public class HadoopShims {
public static boolean isJobFailed(TaskReport report) {
return report.getCurrentStatus()==TIPStatus.FAILED;
}
+
+ public static void unsetConf(Configuration conf, String key) {
+ conf.unset(key);
+ }
}
Modified: pig/branches/branch-0.12/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.12/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java?rev=1561598&r1=1561597&r2=1561598&view=diff
==============================================================================
--- pig/branches/branch-0.12/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java (original)
+++ pig/branches/branch-0.12/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java Mon Jan 27 08:01:58 2014
@@ -18,17 +18,16 @@
package org.apache.pig.backend.hadoop.datastorage;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
import java.util.Enumeration;
import java.util.Iterator;
import java.util.Map;
-import java.util.Properties;
import java.util.Map.Entry;
+import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
import org.apache.pig.ExecType;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce;
+import org.apache.pig.backend.hadoop.executionengine.shims.HadoopShims;
import org.apache.pig.backend.hadoop.executionengine.util.MapRedUtil;
public class ConfigurationUtil {
@@ -80,21 +79,10 @@ public class ConfigurationUtil {
// so build/classes/hadoop-site.xml contains such entry. This prevents some tests from
// successful (They expect those files in hdfs), so we need to unset it in hadoop 23.
// This should go away once MiniMRCluster fix the distributed cache issue.
- Method unsetMethod = null;
- try {
- unsetMethod = localConf.getClass().getMethod("unset", new Class[]{String.class});
- } catch (Exception e) {
- }
- if (unsetMethod!=null) {
- try {
- unsetMethod.invoke(localConf, new Object[]{"mapreduce.job.cache.files"});
- } catch (Exception e) {
- // Should not happen
- }
- }
+ HadoopShims.unsetConf(localConf, "mapreduce.job.cache.files");
}
+ localConf.set(MapRedUtil.FILE_SYSTEM_NAME, "file:///");
Properties props = ConfigurationUtil.toProperties(localConf);
- props.setProperty(MapRedUtil.FILE_SYSTEM_NAME, "file:///");
return props;
}
}