You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2010/04/15 00:38:23 UTC
svn commit: r934230 - in /hadoop/pig/trunk/src/org/apache/pig:
backend/hadoop/datastorage/
backend/hadoop/executionengine/physicalLayer/relationalOperators/
impl/builtin/
Author: daijy
Date: Wed Apr 14 22:38:23 2010
New Revision: 934230
URL: http://svn.apache.org/viewvc?rev=934230&view=rev
Log:
PIG-1338: Pig should exclude hadoop conf in local mode (PIG-1338-newhadoop-1.patch)
Modified:
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POFRJoin.java
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POMergeCogroup.java
hadoop/pig/trunk/src/org/apache/pig/impl/builtin/DefaultIndexableLoader.java
Modified: hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java?rev=934230&r1=934229&r2=934230&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java Wed Apr 14 22:38:23 2010
@@ -25,6 +25,7 @@ import java.util.Properties;
import java.util.Map.Entry;
import org.apache.hadoop.conf.Configuration;
+import org.apache.pig.backend.hadoop.executionengine.util.MapRedUtil;
public class ConfigurationUtil {
@@ -62,4 +63,12 @@ public class ConfigurationUtil {
}
}
+
+ public static Properties getLocalFSProperties() {
+ Configuration localConf = new Configuration(false);
+ localConf.addResource("core-default.xml");
+ Properties props = ConfigurationUtil.toProperties(localConf);
+ props.setProperty(MapRedUtil.FILE_SYSTEM_NAME, "file:///");
+ return props;
+ }
}
Modified: hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POFRJoin.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POFRJoin.java?rev=934230&r1=934229&r2=934230&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POFRJoin.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POFRJoin.java Wed Apr 14 22:38:23 2010
@@ -320,9 +320,7 @@ public class POFRJoin extends PhysicalOp
POLoad ld = new POLoad(new OperatorKey("Repl File Loader", 1L),
replFile);
- Properties props = new Properties();
- props.setProperty(MapRedUtil.FILE_SYSTEM_NAME, "file:///");
-
+ Properties props = ConfigurationUtil.getLocalFSProperties();
PigContext pc = new PigContext(ExecType.LOCAL, props);
ld.setPc(pc);
// We use LocalRearrange Operator to seperate Key and Values
Modified: hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POMergeCogroup.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POMergeCogroup.java?rev=934230&r1=934229&r2=934230&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POMergeCogroup.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POMergeCogroup.java Wed Apr 14 22:38:23 2010
@@ -35,6 +35,7 @@ import org.apache.pig.IndexableLoadFunc;
import org.apache.pig.LoadFunc;
import org.apache.pig.PigException;
import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus;
@@ -378,8 +379,7 @@ public class POMergeCogroup extends Phys
new FileSpec(indexFileName, idxFuncSpec));
// Index file is distributed through Distributed Cache to all mappers. So, read it locally.
- Properties props = new Properties();
- props.setProperty(MapRedUtil.FILE_SYSTEM_NAME, "file:///");
+ Properties props = ConfigurationUtil.getLocalFSProperties();
ld.setPc(new PigContext(ExecType.LOCAL, props));
// Each index entry is read as a pair of split index and a tuple consisting of key.
Modified: hadoop/pig/trunk/src/org/apache/pig/impl/builtin/DefaultIndexableLoader.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/builtin/DefaultIndexableLoader.java?rev=934230&r1=934229&r2=934230&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/impl/builtin/DefaultIndexableLoader.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/impl/builtin/DefaultIndexableLoader.java Wed Apr 14 22:38:23 2010
@@ -110,9 +110,7 @@ public class DefaultIndexableLoader exte
Object firstLeftKey = (keys.size() == 1 ? keys.get(0): keys);
POLoad ld = new POLoad(genKey(), new FileSpec(indexFile, new FuncSpec(indexFileLoadFuncSpec)));
- Properties props = new Properties();
- props.setProperty(MapRedUtil.FILE_SYSTEM_NAME, "file:///");
-
+ Properties props = ConfigurationUtil.getLocalFSProperties();
PigContext pc = new PigContext(ExecType.LOCAL, props);
ld.setPc(pc);
index = new LinkedList<Tuple>();