You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ga...@apache.org on 2010/02/04 01:46:19 UTC
svn commit: r906309 - in /hadoop/pig/trunk: CHANGES.txt
src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
test/org/apache/pig/test/TestLocal2.java
Author: gates
Date: Thu Feb 4 00:46:18 2010
New Revision: 906309
URL: http://svn.apache.org/viewvc?rev=906309&view=rev
Log:
PIG-1154: local mode fails when hadoop config directory is specified in classpath.
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
hadoop/pig/trunk/test/org/apache/pig/test/TestLocal2.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=906309&r1=906308&r2=906309&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Thu Feb 4 00:46:18 2010
@@ -80,6 +80,9 @@
BUG FIXES
+PIG-1154: Local Mode fails when hadoop config directory is specified in
+ classpath (ankit.modi via gates)
+
PIG-1124: Unable to set Custom Job Name using the -Dmapred.job.name parameter (ashutoshc)
PIG-1213: Schema serialization is broken (pradeepkth)
Modified: hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java?rev=906309&r1=906308&r2=906309&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java Thu Feb 4 00:46:18 2010
@@ -18,11 +18,13 @@
package org.apache.pig.backend.hadoop.executionengine;
+import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import java.net.Socket;
import java.net.SocketException;
import java.net.SocketImplFactory;
+import java.net.URL;
import java.util.Collection;
import java.util.List;
import java.util.ArrayList;
@@ -51,10 +53,10 @@
import org.apache.pig.impl.io.FileSpec;
import org.apache.pig.impl.logicalLayer.LogicalPlan;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.LogToPhyTranslationVisitor;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
import org.apache.pig.impl.plan.OperatorKey;
-import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore;
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.tools.pigstats.PigStats;
@@ -64,6 +66,10 @@
public static final String JOB_TRACKER_LOCATION = "mapred.job.tracker";
private static final String FILE_SYSTEM_LOCATION = "fs.default.name";
+ private static final String MAPRED_SITE = "mapred-site.xml";
+ private static final String HDFS_SITE = "hdfs-site.xml";
+ private static final String MAPRED_SYS_DIR = "mapred.system.dir";
+
private final Log log = LogFactory.getLog(getClass());
public static final String LOCAL = "local";
@@ -158,6 +164,30 @@
} else {
properties.setProperty(JOB_TRACKER_LOCATION, LOCAL );
properties.setProperty(FILE_SYSTEM_LOCATION, "file:///");
+
+ Configuration testConf = new Configuration();
+ ClassLoader cl = testConf.getClassLoader();
+ URL mapred_site = cl.getResource( MAPRED_SITE );
+ URL hdfs_site = cl.getResource( HDFS_SITE );
+
+ if( mapred_site != null || hdfs_site != null ) {
+ log.warn( "Passing Hadoop Site Configurations in classpath " +
+ "is not recommended for Local Mode" );
+ }
+
+ // This is one case. Here we check if mapred.system.dir
+ // directory is present. This check causes use to print a nice error
+ String newMapredSystemDir = testConf.get( MAPRED_SYS_DIR, "" );
+ Configuration defaultConf = new Configuration(false);
+ defaultConf.addResource("core-default.xml");
+ defaultConf.addResource("mapred-default.xml");
+ if( defaultConf.get(MAPRED_SYS_DIR, "").compareTo(newMapredSystemDir) != 0 ) {
+ File systemDir = new File(newMapredSystemDir);
+ if( ! systemDir.exists() ) {
+ throw new ExecException( MAPRED_SYS_DIR + ": " + newMapredSystemDir
+ + " mentioned in the configuration does not exist");
+ }
+ }
}
cluster = properties.getProperty(JOB_TRACKER_LOCATION);
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestLocal2.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestLocal2.java?rev=906309&r1=906308&r2=906309&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestLocal2.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestLocal2.java Thu Feb 4 00:46:18 2010
@@ -202,6 +202,44 @@
Assert.assertEquals(count, actualCount);
}
+ @Test
+ public void testLocalInit() throws Exception {
+ File pigFile = new File("script.pig");
+ File siteFile = new File("mapred-site.xml");
+ try {
+ pigFile.createNewFile();
+ int status,status2;
+ status = Util.executeShellCommand("java -cp "+
+ System.getProperty("java.class.path") +
+ " org.apache.pig.Main -x local " + pigFile.getAbsolutePath() );
+
+ String contents = "<?xml version=\"1.0\"?>\n" +
+ "<?xml-stylesheet type=\"text/xsl\" href=\"configuration.xsl\"?>\n" +
+ "<configuration xmlns:xi=\"http://www.w3.org/2001/XInclude\">\n" +
+ " <property>\n" +
+ " <name>mapred.system.dir</name>\n" +
+ " <value>/mapredsystem/hadoop/mapredsystem</value>\n" +
+ " <description>No description</description>\n" +
+ " <final>true</final>\n" +
+ " </property>\n" +
+ "</configuration>\n";
+ assertTrue( siteFile.createNewFile() );
+ PrintStream ps = new PrintStream(siteFile);
+ ps.print(contents);
+ ps.close();
+ status2 = Util.executeShellCommand("java -cp "+
+ System.getProperty("java.class.path") +
+ " org.apache.pig.Main -x local " + pigFile.getAbsolutePath() );
+ assertEquals( "Without a mapred-site.xml pig should just run", 0, status );
+ assertEquals( "With map.system.dir redefined in mapred-site.xml pig " +
+ "should exit", 2, status2 );
+ } finally {
+ if( siteFile.exists() )
+ siteFile.delete();
+ if( pigFile.exists() )
+ pigFile.delete();
+ }
+ }
/***
* For generating a sample dataset
@@ -250,5 +288,4 @@
return TestHelper.createTempFile(data) ;
}
-
}