You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2011/07/19 03:15:02 UTC
svn commit: r1148118 [3/3] - in /pig/branches/branch-0.9: ./ shims/
shims/src/ shims/src/hadoop20/ shims/src/hadoop20/org/
shims/src/hadoop20/org/apache/ shims/src/hadoop20/org/apache/pig/
shims/src/hadoop20/org/apache/pig/backend/ shims/src/hadoop20/o...
Modified: pig/branches/branch-0.9/src/org/apache/pig/impl/io/ReadToEndLoader.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/impl/io/ReadToEndLoader.java?rev=1148118&r1=1148117&r2=1148118&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/impl/io/ReadToEndLoader.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/impl/io/ReadToEndLoader.java Tue Jul 19 01:14:58 2011
@@ -33,6 +33,7 @@ import org.apache.hadoop.mapreduce.TaskA
import org.apache.pig.LoadCaster;
import org.apache.pig.LoadFunc;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit;
+import org.apache.pig.backend.hadoop.executionengine.shims.HadoopShims;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.plan.OperatorKey;
@@ -150,7 +151,7 @@ public class ReadToEndLoader extends Loa
conf = job.getConfiguration();
inputFormat = wrappedLoadFunc.getInputFormat();
try {
- inpSplits = inputFormat.getSplits(new JobContext(conf,
+ inpSplits = inputFormat.getSplits(HadoopShims.createJobContext(conf,
new JobID()));
} catch (InterruptedException e) {
throw new IOException(e);
@@ -167,7 +168,7 @@ public class ReadToEndLoader extends Loa
reader.close();
}
InputSplit curSplit = inpSplits.get(curSplitIndex);
- TaskAttemptContext tAContext = new TaskAttemptContext(conf,
+ TaskAttemptContext tAContext = HadoopShims.createTaskAttemptContext(conf,
new TaskAttemptID());
reader = inputFormat.createRecordReader(curSplit, tAContext);
reader.initialize(curSplit, tAContext);
Modified: pig/branches/branch-0.9/src/org/apache/pig/parser/QueryParserUtils.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/parser/QueryParserUtils.java?rev=1148118&r1=1148117&r2=1148118&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/parser/QueryParserUtils.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/parser/QueryParserUtils.java Tue Jul 19 01:14:58 2011
@@ -97,6 +97,9 @@ public class QueryParserUtils {
static void setHdfsServers(String absolutePath, PigContext pigContext) throws URISyntaxException {
// Get native host
String defaultFS = (String)pigContext.getProperties().get("fs.default.name");
+ if (defaultFS==null)
+ defaultFS = (String)pigContext.getProperties().get("fs.defaultFS");
+
URI defaultFSURI = new URI(defaultFS);
String defaultHost = defaultFSURI.getHost();
if (defaultHost == null) defaultHost = "";
Modified: pig/branches/branch-0.9/src/overview.html
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/overview.html?rev=1148118&r1=1148117&r2=1148118&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/overview.html (original)
+++ pig/branches/branch-0.9/src/overview.html Tue Jul 19 01:14:58 2011
@@ -46,9 +46,9 @@ submitted to Hadoop and monitored by the
{@link org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher}.
<p>
On the backend, each
-{@link org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce.Map},
+{@link org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigGenericMapReduce.Map},
{@link org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigCombiner.Combine}, and
-{@link org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce.Reduce}
+{@link org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigGenericMapReduce.Reduce}
use the pipeline of physical operators constructed in the front end to load, process, and store
data.
Modified: pig/branches/branch-0.9/test/findbugsExcludeFile.xml
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/findbugsExcludeFile.xml?rev=1148118&r1=1148117&r2=1148118&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/findbugsExcludeFile.xml (original)
+++ pig/branches/branch-0.9/test/findbugsExcludeFile.xml Tue Jul 19 01:14:58 2011
@@ -425,4 +425,8 @@
<Method name = "mergeCollection" />
<Bug pattern = "NP_GUARANTEED_DEREF_ON_EXCEPTION_PATH" />
</Match>
+ <Match>
+ <Class name = "org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigGenericMapReduce" />
+ <Bug pattern = "MS_CANNOT_BE_FINAL" />
+ </Match>
</FindBugsFilter>
Modified: pig/branches/branch-0.9/test/org/apache/pig/test/MiniCluster.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/MiniCluster.java?rev=1148118&r1=1148117&r2=1148118&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/MiniCluster.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/MiniCluster.java Tue Jul 19 01:14:58 2011
@@ -1,150 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.pig.test;
-
-import java.io.*;
-import java.util.Properties;
-
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.mapred.MiniMRCluster;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
-
-/**
- * This class builds a single instance of itself with the Singleton
- * design pattern. While building the single instance, it sets up a
- * mini cluster that actually consists of a mini DFS cluster and a
- * mini MapReduce cluster on the local machine and also sets up the
- * environment for Pig to run on top of the mini cluster.
- */
-public class MiniCluster {
- private MiniDFSCluster m_dfs = null;
- private MiniMRCluster m_mr = null;
- private FileSystem m_fileSys = null;
- private JobConf m_conf = null;
-
- private final static MiniCluster INSTANCE = new MiniCluster();
- private static boolean isSetup = true;
-
- private MiniCluster() {
- setupMiniDfsAndMrClusters();
- }
-
- private void setupMiniDfsAndMrClusters() {
- try {
- final int dataNodes = 4; // There will be 4 data nodes
- final int taskTrackers = 4; // There will be 4 task tracker nodes
-
- // Create the configuration hadoop-site.xml file
- File conf_dir = new File(System.getProperty("user.home"), "pigtest/conf/");
- conf_dir.mkdirs();
- File conf_file = new File(conf_dir, "hadoop-site.xml");
-
- conf_file.delete();
-
- // Builds and starts the mini dfs and mapreduce clusters
- Configuration config = new Configuration();
- m_dfs = new MiniDFSCluster(config, dataNodes, true, null);
- m_fileSys = m_dfs.getFileSystem();
- m_mr = new MiniMRCluster(taskTrackers, m_fileSys.getUri().toString(), 1);
-
- // Write the necessary config info to hadoop-site.xml
- m_conf = m_mr.createJobConf();
- m_conf.setInt("mapred.submit.replication", 2);
- m_conf.set("dfs.datanode.address", "0.0.0.0:0");
- m_conf.set("dfs.datanode.http.address", "0.0.0.0:0");
- m_conf.set("mapred.map.max.attempts", "2");
- m_conf.set("mapred.reduce.max.attempts", "2");
- m_conf.writeXml(new FileOutputStream(conf_file));
-
- // Set the system properties needed by Pig
- System.setProperty("cluster", m_conf.get("mapred.job.tracker"));
- System.setProperty("namenode", m_conf.get("fs.default.name"));
- System.setProperty("junit.hadoop.conf", conf_dir.getPath());
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- /**
- * Returns the single instance of class MiniClusterBuilder that
- * represents the resouces for a mini dfs cluster and a mini
- * mapreduce cluster.
- */
- public static MiniCluster buildCluster() {
- if(! isSetup){
- INSTANCE.setupMiniDfsAndMrClusters();
- isSetup = true;
- }
- return INSTANCE;
- }
-
- public void shutDown(){
- INSTANCE.shutdownMiniDfsAndMrClusters();
- }
-
- protected void finalize() {
- shutdownMiniDfsAndMrClusters();
- }
-
- private void shutdownMiniDfsAndMrClusters() {
- isSetup = false;
- try {
- if (m_fileSys != null) { m_fileSys.close(); }
- } catch (IOException e) {
- e.printStackTrace();
- }
- if (m_dfs != null) { m_dfs.shutdown(); }
- if (m_mr != null) { m_mr.shutdown(); }
- m_fileSys = null;
- m_dfs = null;
- m_mr = null;
- }
-
- public Properties getProperties() {
- errorIfNotSetup();
- return ConfigurationUtil.toProperties(m_conf);
- }
-
- public Configuration getConfiguration() {
- return new Configuration(m_conf);
- }
-
- public void setProperty(String name, String value) {
- errorIfNotSetup();
- m_conf.set(name, value);
- }
-
- public FileSystem getFileSystem() {
- errorIfNotSetup();
- return m_fileSys;
- }
-
- /**
- * Throw RunTimeException if isSetup is false
- */
- private void errorIfNotSetup(){
- if(isSetup)
- return;
- String msg = "function called on MiniCluster that has been shutdown";
- throw new RuntimeException(msg);
- }
-}
Added: pig/branches/branch-0.9/test/org/apache/pig/test/MiniGenericCluster.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/MiniGenericCluster.java?rev=1148118&view=auto
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/MiniGenericCluster.java (added)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/MiniGenericCluster.java Tue Jul 19 01:14:58 2011
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.test;
+
+import java.io.*;
+import java.util.Properties;
+
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
+
+/**
+ * This class builds a single instance of itself with the Singleton
+ * design pattern. While building the single instance, it sets up a
+ * mini cluster that actually consists of a mini DFS cluster and a
+ * mini MapReduce cluster on the local machine and also sets up the
+ * environment for Pig to run on top of the mini cluster.
+ *
+ * This class is the base class for MiniCluster, which has slightly
+ * difference among different versions of hadoop. MiniCluster implementation
+ * is located in $PIG_HOME/shims.
+ */
+abstract public class MiniGenericCluster {
+ protected MiniDFSCluster m_dfs = null;
+ protected FileSystem m_fileSys = null;
+ protected Configuration m_conf = null;
+
+ protected final static MiniCluster INSTANCE = new MiniCluster();
+ protected static boolean isSetup = true;
+
+ protected MiniGenericCluster() {
+ setupMiniDfsAndMrClusters();
+ }
+
+ abstract protected void setupMiniDfsAndMrClusters();
+
+ /**
+ * Returns the single instance of class MiniClusterBuilder that
+ * represents the resouces for a mini dfs cluster and a mini
+ * mapreduce cluster.
+ */
+ public static MiniCluster buildCluster() {
+ if(! isSetup){
+ INSTANCE.setupMiniDfsAndMrClusters();
+ isSetup = true;
+ }
+ return INSTANCE;
+ }
+
+ public void shutDown(){
+ INSTANCE.shutdownMiniDfsAndMrClusters();
+ }
+
+ protected void finalize() {
+ shutdownMiniDfsAndMrClusters();
+ }
+
+ protected void shutdownMiniDfsAndMrClusters() {
+ isSetup = false;
+ shutdownMiniDfsClusters();
+ shutdownMiniMrClusters();
+ }
+
+ protected void shutdownMiniDfsClusters() {
+ try {
+ if (m_fileSys != null) { m_fileSys.close(); }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ if (m_dfs != null) { m_dfs.shutdown(); }
+ m_fileSys = null;
+ m_dfs = null;
+ }
+
+ abstract protected void shutdownMiniMrClusters();
+
+ public Properties getProperties() {
+ errorIfNotSetup();
+ return ConfigurationUtil.toProperties(m_conf);
+ }
+
+ public Configuration getConfiguration() {
+ return new Configuration(m_conf);
+ }
+
+ public void setProperty(String name, String value) {
+ errorIfNotSetup();
+ m_conf.set(name, value);
+ }
+
+ public FileSystem getFileSystem() {
+ errorIfNotSetup();
+ return m_fileSys;
+ }
+
+ /**
+ * Throw RunTimeException if isSetup is false
+ */
+ private void errorIfNotSetup(){
+ if(isSetup)
+ return;
+ String msg = "function called on MiniCluster that has been shutdown";
+ throw new RuntimeException(msg);
+ }
+}
Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestGrunt.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestGrunt.java?rev=1148118&r1=1148117&r2=1148118&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestGrunt.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestGrunt.java Tue Jul 19 01:14:58 2011
@@ -522,7 +522,7 @@ public class TestGrunt extends TestCase
@Test
public void testIllustrateScript() throws Throwable {
- PigServer server = new PigServer(ExecType.LOCAL, cluster.getProperties());
+ PigServer server = new PigServer(ExecType.LOCAL, new Properties());
PigContext context = server.getPigContext();
String strCmd = "illustrate -script "
@@ -538,7 +538,7 @@ public class TestGrunt extends TestCase
@Test
public void testIllustrateScript2() throws Throwable {
- PigServer server = new PigServer(ExecType.LOCAL, cluster.getProperties());
+ PigServer server = new PigServer(ExecType.LOCAL, new Properties());
PigContext context = server.getPigContext();
String strCmd = "illustrate -script "
@@ -554,7 +554,7 @@ public class TestGrunt extends TestCase
@Test
public void testIllustrateScript3() throws Throwable {
- PigServer server = new PigServer(ExecType.LOCAL, cluster.getProperties());
+ PigServer server = new PigServer(ExecType.LOCAL, new Properties());
PigContext context = server.getPigContext();
String strCmd = "illustrate -script "
@@ -571,7 +571,7 @@ public class TestGrunt extends TestCase
@Test
public void testIllustrateScript4() throws Throwable {
// empty line/field test
- PigServer server = new PigServer(ExecType.LOCAL, cluster.getProperties());
+ PigServer server = new PigServer(ExecType.LOCAL, new Properties());
PigContext context = server.getPigContext();
String strCmd = "illustrate -script "
@@ -588,7 +588,7 @@ public class TestGrunt extends TestCase
@Test
public void testIllustrateScript5() throws Throwable {
// empty line/field test
- PigServer server = new PigServer(ExecType.LOCAL, cluster.getProperties());
+ PigServer server = new PigServer(ExecType.LOCAL, new Properties());
PigContext context = server.getPigContext();
String strCmd = "illustrate -script "
@@ -605,7 +605,7 @@ public class TestGrunt extends TestCase
@Test
public void testIllustrateScript6() throws Throwable {
// empty line/field test
- PigServer server = new PigServer(ExecType.LOCAL, cluster.getProperties());
+ PigServer server = new PigServer(ExecType.LOCAL, new Properties());
PigContext context = server.getPigContext();
String strCmd = "illustrate -script "
@@ -622,7 +622,7 @@ public class TestGrunt extends TestCase
@Test
public void testIllustrateScript7() throws Throwable {
// empty line/field test
- PigServer server = new PigServer(ExecType.LOCAL, cluster.getProperties());
+ PigServer server = new PigServer(ExecType.LOCAL, new Properties());
PigContext context = server.getPigContext();
String strCmd = "illustrate -script "
@@ -1345,4 +1345,4 @@ public class TestGrunt extends TestCase
Util.checkMessageInException(e, errMsg);
}
}
-}
\ No newline at end of file
+}
Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestTypedMap.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestTypedMap.java?rev=1148118&r1=1148117&r2=1148118&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestTypedMap.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestTypedMap.java Tue Jul 19 01:14:58 2011
@@ -23,6 +23,7 @@ import java.io.File;
import java.io.IOException;
import java.util.Iterator;
import java.util.Map;
+import java.util.Properties;
import junit.framework.Assert;
@@ -43,7 +44,6 @@ import org.junit.BeforeClass;
import org.junit.Test;
public class TestTypedMap {
- static MiniCluster cluster ;
private String tmpDirName = System.getProperty("java.io.tmpdir") + "/pigtest/tmp";
@Before
@@ -71,19 +71,9 @@ public class TestTypedMap {
}
}
- @BeforeClass
- public static void oneTimeSetup() throws IOException, Exception {
- cluster = MiniCluster.buildCluster();
- }
-
- @AfterClass
- public static void oneTimeTearDown() throws Exception {
- cluster.shutDown();
- }
-
@Test
public void testSimpleLoad() throws IOException, ParserException {
- PigServer pig = new PigServer(ExecType.LOCAL, cluster.getProperties());
+ PigServer pig = new PigServer(ExecType.LOCAL, new Properties());
String[] input = {
"[key#1,key2#2]",
"[key#2]",
@@ -120,7 +110,7 @@ public class TestTypedMap {
@Test
public void testSimpleMapKeyLookup() throws IOException, ParserException {
- PigServer pig = new PigServer(ExecType.LOCAL, cluster.getProperties());
+ PigServer pig = new PigServer(ExecType.LOCAL, new Properties());
String[] input = {
"[key#1,key2#2]",
"[key#2]",
@@ -151,7 +141,7 @@ public class TestTypedMap {
@Test
public void testSimpleMapCast() throws IOException, ParserException {
- PigServer pig = new PigServer(ExecType.LOCAL, cluster.getProperties());
+ PigServer pig = new PigServer(ExecType.LOCAL, new Properties());
String[] input = {
"[key#1,key2#2]",
"[key#2]",
@@ -189,7 +179,7 @@ public class TestTypedMap {
@Test
public void testComplexLoad() throws IOException, ParserException {
- PigServer pig = new PigServer(ExecType.LOCAL, cluster.getProperties());
+ PigServer pig = new PigServer(ExecType.LOCAL, new Properties());
String[] input = {
"[key#{(1,2),(1,3)},134#]",
"[key2#]",
@@ -223,7 +213,7 @@ public class TestTypedMap {
@Test
public void testComplexCast() throws IOException, ParserException {
- PigServer pig = new PigServer(ExecType.LOCAL, cluster.getProperties());
+ PigServer pig = new PigServer(ExecType.LOCAL, new Properties());
String[] input = {
"[key#{(1,2),(1,3)},134#]",
"[key2#]",
@@ -258,7 +248,7 @@ public class TestTypedMap {
@Test
public void testComplexCast2() throws IOException, ParserException {
- PigServer pig = new PigServer(ExecType.LOCAL, cluster.getProperties());
+ PigServer pig = new PigServer(ExecType.LOCAL, new Properties());
String[] input = {
"[key#1,key2#2]",
};
@@ -288,7 +278,7 @@ public class TestTypedMap {
@Test
public void testUnTypedMap() throws IOException, ParserException {
- PigServer pig = new PigServer(ExecType.LOCAL, cluster.getProperties());
+ PigServer pig = new PigServer(ExecType.LOCAL, new Properties());
String[] input = {
"[key#1,key2#2]",
};
@@ -317,7 +307,7 @@ public class TestTypedMap {
@Test
public void testOrderBy() throws IOException, ParserException {
- PigServer pig = new PigServer(ExecType.LOCAL, cluster.getProperties());
+ PigServer pig = new PigServer(ExecType.LOCAL, new Properties());
String[] input = {
"[key#1,key1#2]",
"[key#2,key3#2]",