You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2014/11/03 18:56:36 UTC
svn commit: r1636407 - in /pig/trunk: ./
src/org/apache/pig/backend/hadoop/datastorage/
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/
test/org/apache/pig/test/
Author: daijy
Date: Mon Nov 3 17:56:36 2014
New Revision: 1636407
URL: http://svn.apache.org/r1636407
Log:
PIG-4257: Fix several e2e tests on secure cluster
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/WeightedRangePartitioner.java
pig/trunk/test/org/apache/pig/test/TestMapSideCogroup.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1636407&r1=1636406&r2=1636407&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Mon Nov 3 17:56:36 2014
@@ -112,6 +112,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-4257: Fix several e2e tests on secure cluster (daijy)
+
PIG-4261: Skip shipping local resources in tez local mode (daijy)
PIG-4182: e2e tests Scripting_[1-12] fail on Windows (daijy)
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java?rev=1636407&r1=1636406&r2=1636407&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java Mon Nov 3 17:56:36 2014
@@ -79,7 +79,11 @@ public class ConfigurationUtil {
localConf = new Configuration(false);
localConf.addResource("core-default.xml");
} else {
- localConf = new Configuration(true);
+ if (PigMapReduce.sJobContext!=null) {
+ localConf = PigMapReduce.sJobContext.getConfiguration();
+ } else {
+ localConf = new Configuration(true);
+ }
// It's really hacky, try to get unit test working under hadoop 23.
// Hadoop23 MiniMRCluster currently need setup Distributed cache before start,
// so build/classes/hadoop-site.xml contains such entry. This prevents some tests from
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/WeightedRangePartitioner.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/WeightedRangePartitioner.java?rev=1636407&r1=1636406&r2=1636407&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/WeightedRangePartitioner.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/WeightedRangePartitioner.java Mon Nov 3 17:56:36 2014
@@ -31,6 +31,7 @@ import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.Partitioner;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.backend.hadoop.HDataType;
+import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce;
import org.apache.pig.backend.hadoop.executionengine.util.MapRedUtil;
import org.apache.pig.data.DataBag;
@@ -109,7 +110,7 @@ public class WeightedRangePartitioner ex
Map<String, Object> quantileMap = null;
Configuration conf;
if (!pigContext.getExecType().isLocal()) {
- conf = new Configuration(true);
+ conf = ConfigurationUtil.toConfiguration(pigContext.getProperties());
} else {
conf = new Configuration(false);
}
Modified: pig/trunk/test/org/apache/pig/test/TestMapSideCogroup.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestMapSideCogroup.java?rev=1636407&r1=1636406&r2=1636407&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestMapSideCogroup.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestMapSideCogroup.java Mon Nov 3 17:56:36 2014
@@ -479,7 +479,7 @@ public class TestMapSideCogroup {
@Override
public void initialize(Configuration conf) throws IOException {
- is = FileSystem.get(conf).open(new Path(loc));
+ is = FileSystem.get(new Path(loc).toUri(), conf).open(new Path(loc));
}
@Override