You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2012/09/24 20:06:50 UTC
svn commit: r1389502 - in /pig/trunk: ./ ivy/
shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/
shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/
shims/test/hadoop23/org/apache/pig/test/ src/org/apache/pig/ba...
Author: daijy
Date: Mon Sep 24 18:06:49 2012
New Revision: 1389502
URL: http://svn.apache.org/viewvc?rev=1389502&view=rev
Log:
PIG-2791: Pig does not work with ViewFileSystem
Modified:
pig/trunk/CHANGES.txt
pig/trunk/build.xml
pig/trunk/ivy.xml
pig/trunk/ivy/libraries.properties
pig/trunk/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java
pig/trunk/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java
pig/trunk/shims/test/hadoop23/org/apache/pig/test/MiniCluster.java
pig/trunk/src/org/apache/pig/backend/datastorage/DataStorage.java
pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/HDataStorage.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java
pig/trunk/test/org/apache/pig/test/TestBuiltin.java
pig/trunk/test/org/apache/pig/test/TestJobSubmission.java
pig/trunk/test/org/apache/pig/test/TestPigRunner.java
pig/trunk/test/org/apache/pig/test/TestScriptUDF.java
pig/trunk/test/org/apache/pig/test/Util.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1389502&r1=1389501&r2=1389502&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Mon Sep 24 18:06:49 2012
@@ -281,6 +281,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-2791: Pig does not work with ViewFileSystem (rohini via daijy)
+
PIG-2833: org.apache.pig.pigunit.pig.PigServer does not initialize set default log level of pigContext (cheolsoo via jcoveney)
PIG-2744: Handle Pig command line with XML special characters (lulynn_2008 via daijy)
Modified: pig/trunk/build.xml
URL: http://svn.apache.org/viewvc/pig/trunk/build.xml?rev=1389502&r1=1389501&r2=1389502&view=diff
==============================================================================
--- pig/trunk/build.xml (original)
+++ pig/trunk/build.xml Mon Sep 24 18:06:49 2012
@@ -486,7 +486,11 @@
<copy file="${basedir}/test/hbase-site.xml" tofile="${test.build.classes}/hbase-site.xml"/>
- <ivy:cachepath pathid="mr-apps-test.classpath" conf="test" />
+ <ivy:cachepath pathid="mr-apps-test-ivy.classpath" conf="test" />
+ <path id="mr-apps-test.classpath">
+ <pathelement path="${clover.jar}"/>
+ <path refid="mr-apps-test-ivy.classpath"/>
+ </path>
<property name="mr-apps-classpath" refid="mr-apps-test.classpath" />
<echo file="${test.build.classes}/mrapp-generated-classpath" message="${mr-apps-classpath}" />
</target>
Modified: pig/trunk/ivy.xml
URL: http://svn.apache.org/viewvc/pig/trunk/ivy.xml?rev=1389502&r1=1389501&r2=1389502&view=diff
==============================================================================
--- pig/trunk/ivy.xml (original)
+++ pig/trunk/ivy.xml Mon Sep 24 18:06:49 2012
@@ -69,6 +69,8 @@
conf="hadoop23->master"/>
<dependency org="commons-el" name="commons-el" rev="${commons-el.version}"
conf="compile->master"/>
+ <dependency org="commons-io" name="commons-io" rev="${commons-io.version}"
+ conf="compile->master"/>
<dependency org="org.apache.httpcomponents" name="httpclient" rev="${httpcomponents.version}"
conf="compile->master"/>
<dependency org="org.apache.httpcomponents" name="httpcore" rev="${httpcomponents.version}"
@@ -87,6 +89,12 @@
conf="hadoop23->master"/>
<dependency org="org.mortbay.jetty" name="jetty-util" rev="${jetty-util.version}"
conf="hadoop23->master"/>
+ <dependency org="javax.inject" name="javax.inject" rev="${javax-inject.version}"
+ conf="hadoop23->master"/>
+ <dependency org="javax.xml.bind" name="jaxb-api" rev="${jaxb-api.version}"
+ conf="hadoop23->master"/>
+ <dependency org="com.sun.xml.bind" name="jaxb-impl" rev="${jaxb-impl.version}"
+ conf="hadoop23->master"/>
<dependency org="com.google.inject" name="guice" rev="${guice.version}"
conf="hadoop23->master"/>
<dependency org="com.google.inject.extensions" name="guice-servlet" rev="${guice-servlet.version}"
Modified: pig/trunk/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/pig/trunk/ivy/libraries.properties?rev=1389502&r1=1389501&r2=1389502&view=diff
==============================================================================
--- pig/trunk/ivy/libraries.properties (original)
+++ pig/trunk/ivy/libraries.properties Mon Sep 24 18:06:49 2012
@@ -20,6 +20,7 @@ avro.version=1.5.3
commons-beanutils.version=1.7.0
commons-cli.version=1.0
commons-codec.version=1.4
+commons-io.version=2.3
commons-el.version=1.0
commons-logging.version=1.1.1
commons-lang.version=2.4
@@ -36,15 +37,18 @@ guava.version=11.0
jersey-core.version=1.8
hadoop-core.version=1.0.0
hadoop-test.version=1.0.0
-hadoop-common.version=0.23.1
-hadoop-hdfs.version=0.23.1
-hadoop-mapreduce.version=0.23.1
+hadoop-common.version=2.0.0-alpha
+hadoop-hdfs.version=2.0.0-alpha
+hadoop-mapreduce.version=2.0.0-alpha
hbase.version=0.90.0
hsqldb.version=1.8.0.10
hive.version=0.8.0
httpcomponents.version=4.1
-jackson.version=1.7.3
+jackson.version=1.8.8
javacc.version=4.2
+javax-inject.version=1
+jaxb-api.version=2.2.2
+jaxb-impl.version=2.2.3-1
jdeb.version=0.8
jdiff.version=1.0.9
jetty.version=6.1.26
@@ -71,8 +75,8 @@ zookeeper.version=3.3.3
servlet.version=4.0.6
servlet-api.version=2.5
protobuf-java.version=2.4.0a
-guice.version=2.0
-guice-servlet.version=2.0
+guice.version=3.0
+guice-servlet.version=3.0
aopalliance.version=1.0
jsr311-api.version=1.1.1
mockito.version=1.8.4
Modified: pig/trunk/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/pig/trunk/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java?rev=1389502&r1=1389501&r2=1389502&view=diff
==============================================================================
--- pig/trunk/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java (original)
+++ pig/trunk/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java Mon Sep 24 18:06:49 2012
@@ -20,6 +20,8 @@ package org.apache.pig.backend.hadoop.ex
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.jobcontrol.Job;
import org.apache.hadoop.mapred.jobcontrol.JobControl;
import org.apache.hadoop.mapreduce.JobContext;
@@ -93,4 +95,8 @@ public class HadoopShims {
public static JobControl newJobControl(String groupName, int timeToSleep) {
return new PigJobControl(groupName, timeToSleep);
}
+
+ public static long getDefaultBlockSize(FileSystem fs, Path path) {
+ return fs.getDefaultBlockSize();
+ }
}
Modified: pig/trunk/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/pig/trunk/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java?rev=1389502&r1=1389501&r2=1389502&view=diff
==============================================================================
--- pig/trunk/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java (original)
+++ pig/trunk/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java Mon Sep 24 18:06:49 2012
@@ -20,6 +20,8 @@ package org.apache.pig.backend.hadoop.ex
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.jobcontrol.Job;
import org.apache.hadoop.mapred.jobcontrol.JobControl;
@@ -99,4 +101,8 @@ public class HadoopShims {
public static JobControl newJobControl(String groupName, int timeToSleep) {
return new PigJobControl(groupName, timeToSleep);
}
+
+ public static long getDefaultBlockSize(FileSystem fs, Path path) {
+ return fs.getDefaultBlockSize(path);
+ }
}
Modified: pig/trunk/shims/test/hadoop23/org/apache/pig/test/MiniCluster.java
URL: http://svn.apache.org/viewvc/pig/trunk/shims/test/hadoop23/org/apache/pig/test/MiniCluster.java?rev=1389502&r1=1389501&r2=1389502&view=diff
==============================================================================
--- pig/trunk/shims/test/hadoop23/org/apache/pig/test/MiniCluster.java (original)
+++ pig/trunk/shims/test/hadoop23/org/apache/pig/test/MiniCluster.java Mon Sep 24 18:06:49 2012
@@ -69,6 +69,9 @@ public class MiniCluster extends MiniGen
m_dfs = new MiniDFSCluster(config, dataNodes, true, null);
m_fileSys = m_dfs.getFileSystem();
m_dfs_conf = m_dfs.getConfiguration(0);
+
+ //Create user home directory
+ m_fileSys.mkdirs(m_fileSys.getWorkingDirectory());
m_mr = new MiniMRYarnCluster("PigMiniCluster", taskTrackers);
m_mr.init(m_dfs_conf);
Modified: pig/trunk/src/org/apache/pig/backend/datastorage/DataStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/datastorage/DataStorage.java?rev=1389502&r1=1389501&r2=1389502&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/datastorage/DataStorage.java (original)
+++ pig/trunk/src/org/apache/pig/backend/datastorage/DataStorage.java Mon Sep 24 18:06:49 2012
@@ -30,7 +30,6 @@ import java.io.IOException;
public interface DataStorage {
- public static final String DEFAULT_REPLICATION_FACTOR_KEY = "pig.default.replication.factor";
public static final String USED_BYTES_KEY = "pig.used.bytes";
public static final String RAW_CAPACITY_KEY = "pig.raw.capacity.bytes"; // replication is disregarded
public static final String RAW_USED_KEY = "pig.raw.used.capacity.bytes"; // replication is disregarded
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/HDataStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/HDataStorage.java?rev=1389502&r1=1389501&r2=1389502&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/HDataStorage.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/HDataStorage.java Mon Sep 24 18:06:49 2012
@@ -74,9 +74,6 @@ public class HDataStorage implements Dat
} catch (IOException e) {
throw new RuntimeException("Failed to create DataStorage", e);
}
- short defaultReplication = fs.getDefaultReplication();
- properties.setProperty(DEFAULT_REPLICATION_FACTOR_KEY,
- Short.valueOf(defaultReplication).toString());
}
public void close() throws IOException {
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java?rev=1389502&r1=1389501&r2=1389502&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java Mon Sep 24 18:06:49 2012
@@ -1271,8 +1271,9 @@ public class MRCompiler extends PhyPlanV
InputFormat inf = loader.getInputFormat();
List<InputSplit> splits = inf.getSplits(HadoopShims.cloneJobContext(job));
List<List<InputSplit>> results = MapRedUtil
- .getCombinePigSplits(splits, fs
- .getDefaultBlockSize(), conf);
+ .getCombinePigSplits(splits,
+ HadoopShims.getDefaultBlockSize(fs, path),
+ conf);
numFiles += results.size();
} else {
List<MapReduceOper> preds = MRPlan.getPredecessors(mro);
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java?rev=1389502&r1=1389501&r2=1389502&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java Mon Sep 24 18:06:49 2012
@@ -225,7 +225,7 @@ public class PigInputFormat extends Inpu
Path path = new Path(inputs.get(i).getFileName());
FileSystem fs;
-
+ boolean isFsPath = true;
try {
fs = path.getFileSystem(conf);
} catch (Exception e) {
@@ -235,6 +235,7 @@ public class PigInputFormat extends Inpu
// getting the file system. That's
// ok, we just use the dfs in that case.
fs = new Path("/").getFileSystem(conf);
+ isFsPath = false;
}
// if the execution is against Mapred DFS, set
@@ -274,7 +275,9 @@ public class PigInputFormat extends Inpu
HadoopShims.createJobContext(inputSpecificJob.getConfiguration(),
jobcontext.getJobID()));
List<InputSplit> oneInputPigSplits = getPigSplits(
- oneInputSplits, i, inpTargets.get(i), fs.getDefaultBlockSize(), combinable, confClone);
+ oneInputSplits, i, inpTargets.get(i),
+ HadoopShims.getDefaultBlockSize(fs, isFsPath? path: fs.getWorkingDirectory()),
+ combinable, confClone);
splits.addAll(oneInputPigSplits);
} catch (ExecException ee) {
throw ee;
Modified: pig/trunk/test/org/apache/pig/test/TestBuiltin.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestBuiltin.java?rev=1389502&r1=1389501&r2=1389502&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestBuiltin.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestBuiltin.java Mon Sep 24 18:06:49 2012
@@ -192,7 +192,7 @@ public class TestBuiltin {
@Before
public void setUp() throws Exception {
- pigServer = new PigServer(ExecType.LOCAL, cluster.getProperties());
+ pigServer = new PigServer(ExecType.LOCAL, new Properties());
pigServer.setValidateEachStatement(true);
// First set up data structs for "base" SUM, MIN and MAX and AVG.
// The allowed input and expected output data structs for
Modified: pig/trunk/test/org/apache/pig/test/TestJobSubmission.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestJobSubmission.java?rev=1389502&r1=1389501&r2=1389502&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestJobSubmission.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestJobSubmission.java Mon Sep 24 18:06:49 2012
@@ -536,7 +536,7 @@ public class TestJobSubmission {
@Test
public void testReducerNumEstimation() throws Exception{
// skip this test for 23 until HBASE-4850
- if (Util.isHadoop23())
+ if (Util.isHadoop23() || Util.isHadoop2_0())
return;
// use the estimation
Configuration conf = cluster.getConfiguration();
Modified: pig/trunk/test/org/apache/pig/test/TestPigRunner.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigRunner.java?rev=1389502&r1=1389501&r2=1389502&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPigRunner.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPigRunner.java Mon Sep 24 18:06:49 2012
@@ -580,7 +580,7 @@ public class TestPigRunner {
@Test
public void classLoaderTest() throws Exception {
// Skip in hadoop 23 test, see PIG-2449
- if (Util.isHadoop23())
+ if (Util.isHadoop23() || Util.isHadoop2_0())
return;
PrintWriter w = new PrintWriter(new FileWriter(PIG_FILE));
w.println("register test/org/apache/pig/test/data/pigtestloader.jar");
Modified: pig/trunk/test/org/apache/pig/test/TestScriptUDF.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestScriptUDF.java?rev=1389502&r1=1389501&r2=1389502&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestScriptUDF.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestScriptUDF.java Mon Sep 24 18:06:49 2012
@@ -159,7 +159,7 @@ public class TestScriptUDF{
@Test
public void testPythonNestedImport() throws Exception {
// Skip for hadoop 23 until PIG-2433 fixed
- if (Util.isHadoop23())
+ if (Util.isHadoop23() || Util.isHadoop2_0())
return;
String[] scriptA = {
Modified: pig/trunk/test/org/apache/pig/test/Util.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/Util.java?rev=1389502&r1=1389501&r2=1389502&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/Util.java (original)
+++ pig/trunk/test/org/apache/pig/test/Util.java Mon Sep 24 18:06:49 2012
@@ -533,6 +533,14 @@ public class Util {
}
}
+ static private String getMkDirCommandForHadoop2_0(String fileName) {
+ if (Util.isHadoop23() || Util.isHadoop2_0()) {
+ Path parentDir = new Path(fileName).getParent();
+ String mkdirCommand = parentDir.getName().isEmpty() ? "" : "fs -mkdir -p " + parentDir + "\n";
+ return mkdirCommand;
+ }
+ return "";
+ }
/**
* Utility method to copy a file form local filesystem to the dfs on
@@ -544,7 +552,7 @@ public class Util {
*/
static public void copyFromLocalToCluster(MiniCluster cluster, String localFileName, String fileNameOnCluster) throws IOException {
PigServer ps = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
- String script = "fs -put " + localFileName + " " + fileNameOnCluster;
+ String script = getMkDirCommandForHadoop2_0(fileNameOnCluster) + "fs -put " + localFileName + " " + fileNameOnCluster;
GruntParser parser = new GruntParser(new StringReader(script));
parser.setInteractive(false);
@@ -559,7 +567,7 @@ public class Util {
static public void copyFromLocalToLocal(String fromLocalFileName,
String toLocalFileName) throws IOException {
PigServer ps = new PigServer(ExecType.LOCAL, new Properties());
- String script = "fs -cp " + fromLocalFileName + " " + toLocalFileName;
+ String script = getMkDirCommandForHadoop2_0(toLocalFileName) + "fs -cp " + fromLocalFileName + " " + toLocalFileName;
new File(toLocalFileName).deleteOnExit();
@@ -1180,4 +1188,11 @@ public class Util {
private static void assertConfLong(Configuration conf, String param, long expected) {
assertEquals("Unexpected value found in configs for " + param, expected, conf.getLong(param, -1));
}
+
+ public static boolean isHadoop2_0() {
+ String version = org.apache.hadoop.util.VersionInfo.getVersion();
+ if (version.matches("\\b2\\.0\\..+"))
+ return true;
+ return false;
+ }
}