You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ga...@apache.org on 2008/02/27 01:52:08 UTC
svn commit: r631443 [9/10] - in /incubator/pig/branches/types: ./
lib-src/bzip2/org/apache/tools/bzip2r/ lib-src/shock/org/apache/pig/shock/
lib/ scripts/ src/org/apache/pig/ src/org/apache/pig/backend/
src/org/apache/pig/backend/datastorage/ src/org/a...
Modified: incubator/pig/branches/types/src/org/apache/pig/tools/streams/StreamGenerator.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/tools/streams/StreamGenerator.java?rev=631443&r1=631442&r2=631443&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/tools/streams/StreamGenerator.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/tools/streams/StreamGenerator.java Tue Feb 26 16:51:49 2008
@@ -96,42 +96,42 @@
public void actionPerformed(ActionEvent event) {
if (out == null){
-
- try{
- String fileName = fileField.getText();
+
+ try{
+ String fileName = fileField.getText();
String format = formatField.getText();
- out = new PrintWriter(fileName);
- String[] formats = format.split(",");
- this.formats = new int[formats.length];
- for (int i=0; i<formats.length; i++){
- this.formats[i] = (int)Math.pow(10,Integer.parseInt(formats[i]));
- }
- }catch(Exception e){
- new Dialog(generatorFrame,"Input not well formed");
- }
-
- //First time
- fileField.setEditable(false);
- formatField.setEditable(false);
+ out = new PrintWriter(fileName);
+ String[] formats = format.split(",");
+ this.formats = new int[formats.length];
+ for (int i=0; i<formats.length; i++){
+ this.formats[i] = (int)Math.pow(10,Integer.parseInt(formats[i]));
+ }
+ }catch(Exception e){
+ new Dialog(generatorFrame,"Input not well formed");
+ }
+
+ //First time
+ fileField.setEditable(false);
+ formatField.setEditable(false);
}
int numTuples=0;
try{
- numTuples = Integer.parseInt(numberField.getText());
+ numTuples = Integer.parseInt(numberField.getText());
}catch(Exception e){
- new Dialog(generatorFrame,"Input not well formed");
- }
+ new Dialog(generatorFrame,"Input not well formed");
+ }
for (int i=0; i<numTuples; i++){
- for (int j=0; j<formats.length; j++){
- out.print(random.nextInt(formats[j]));
- if (j==formats.length-1)
- out.println("");
- else
- out.print("\t");
- }
+ for (int j=0; j<formats.length; j++){
+ out.print(random.nextInt(formats[j]));
+ if (j==formats.length-1)
+ out.println("");
+ else
+ out.print("\t");
+ }
}
out.flush();
Modified: incubator/pig/branches/types/src/org/apache/pig/tools/timer/PerformanceTimer.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/tools/timer/PerformanceTimer.java?rev=631443&r1=631442&r2=631443&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/tools/timer/PerformanceTimer.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/tools/timer/PerformanceTimer.java Tue Feb 26 16:51:49 2008
@@ -34,9 +34,9 @@
*/
public void start()
{
- mStartedAt = System.nanoTime();
- mStarts++;
- mState = State.RUNNING;
+ mStartedAt = System.nanoTime();
+ mStarts++;
+ mState = State.RUNNING;
}
/**
@@ -44,8 +44,8 @@
*/
public void stop()
{
- mState = State.STOPPED;
- mNanosecs += System.nanoTime() - mStartedAt;
+ mState = State.STOPPED;
+ mNanosecs += System.nanoTime() - mStartedAt;
}
/**
@@ -55,22 +55,22 @@
*/
public void print(PrintStream out)
{
- if (mStarts == 0) {
- out.println(mName + " never started.");
- return;
- }
-
- if (mState == State.RUNNING) out.print("WARNING: timer still running! ");
- out.print(mName + ": ");
- double t = mNanosecs / 1000000000.0;
- out.print(t);
- out.print(". Run ");
- out.print(mStarts);
- out.print(" times, average run time ");
- long avg = mNanosecs / mStarts;
- t = avg / 1000000000.0;
- out.print(t);
- out.println(".");
+ if (mStarts == 0) {
+ out.println(mName + " never started.");
+ return;
+ }
+
+ if (mState == State.RUNNING) out.print("WARNING: timer still running! ");
+ out.print(mName + ": ");
+ double t = mNanosecs / 1000000000.0;
+ out.print(t);
+ out.print(". Run ");
+ out.print(mStarts);
+ out.print(" times, average run time ");
+ long avg = mNanosecs / mStarts;
+ t = avg / 1000000000.0;
+ out.print(t);
+ out.println(".");
}
/**
@@ -78,10 +78,10 @@
*/
PerformanceTimer(String name)
{
- mNanosecs = 0;
- mStarts = 0;
- mName = name;
- mState = State.STOPPED;
+ mNanosecs = 0;
+ mStarts = 0;
+ mName = name;
+ mState = State.STOPPED;
}
Modified: incubator/pig/branches/types/src/org/apache/pig/tools/timer/PerformanceTimerFactory.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/tools/timer/PerformanceTimerFactory.java?rev=631443&r1=631442&r2=631443&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/tools/timer/PerformanceTimerFactory.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/tools/timer/PerformanceTimerFactory.java Tue Feb 26 16:51:49 2008
@@ -42,10 +42,10 @@
*/
public static PerformanceTimerFactory getPerfTimerFactory()
{
- if (self == null) {
- self = new PerformanceTimerFactory();
- }
- return self;
+ if (self == null) {
+ self = new PerformanceTimerFactory();
+ }
+ return self;
}
/**
@@ -56,12 +56,12 @@
*/
public PerformanceTimer getTimer(String name)
{
- PerformanceTimer timer = mTimers.get(name);
- if (timer == null) {
- timer = new PerformanceTimer(name);
- mTimers.put(name, timer);
- }
- return timer;
+ PerformanceTimer timer = mTimers.get(name);
+ if (timer == null) {
+ timer = new PerformanceTimer(name);
+ mTimers.put(name, timer);
+ }
+ return timer;
}
/**
@@ -70,23 +70,23 @@
*/
public void dumpTimers(PrintStream out)
{
- Collection<PerformanceTimer> c = mTimers.values();
- Iterator<PerformanceTimer> i = c.iterator();
- while (i.hasNext()) {
- i.next().print(out);
- }
+ Collection<PerformanceTimer> c = mTimers.values();
+ Iterator<PerformanceTimer> i = c.iterator();
+ while (i.hasNext()) {
+ i.next().print(out);
+ }
}
public void dumpTimers()
{
- dumpTimers(System.out);
+ dumpTimers(System.out);
}
private static PerformanceTimerFactory self = null;
private PerformanceTimerFactory()
{
- mTimers = new HashMap<String, PerformanceTimer>();
+ mTimers = new HashMap<String, PerformanceTimer>();
}
HashMap<String, PerformanceTimer> mTimers;
Added: incubator/pig/branches/types/test/org/apache/pig/test/MiniCluster.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/MiniCluster.java?rev=631443&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/MiniCluster.java (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/MiniCluster.java Tue Feb 26 16:51:49 2008
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.test;
+
+import java.io.*;
+
+import org.apache.hadoop.dfs.MiniDFSCluster;
+import org.apache.hadoop.mapred.MiniMRCluster;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.fs.FileSystem;
+
+/**
+ * This class builds a single instance of itself with the Singleton
+ * design pattern. While building the single instance, it sets up a
+ * mini cluster that actually consists of a mini DFS cluster and a
+ * mini MapReduce cluster on the local machine and also sets up the
+ * environment for Pig to run on top of the mini cluster.
+ */
+public class MiniCluster {
+ private MiniDFSCluster m_dfs = null;
+ private MiniMRCluster m_mr = null;
+ private FileSystem m_fileSys = null;
+ private JobConf m_conf = null;
+
+ private final static MiniCluster INSTANCE = new MiniCluster();
+
+ private MiniCluster() {
+ setupMiniDfsAndMrClusters();
+ }
+
+ private void setupMiniDfsAndMrClusters() {
+ try {
+ final int dataNodes = 4; // There will be 4 data nodes
+ final int taskTrackers = 4; // There will be 4 task tracker nodes
+ Configuration config = new Configuration();
+
+ // Builds and starts the mini dfs and mapreduce clusters
+ m_dfs = new MiniDFSCluster(config, dataNodes, true, null);
+ m_fileSys = m_dfs.getFileSystem();
+ m_mr = new MiniMRCluster(taskTrackers, m_fileSys.getName(), 1);
+
+ // Create the configuration hadoop-site.xml file
+ File conf_dir = new File(System.getProperty("user.home"), "pigtest/conf/");
+ conf_dir.mkdirs();
+ File conf_file = new File(conf_dir, "hadoop-site.xml");
+
+ // Write the necessary config info to hadoop-site.xml
+ m_conf = m_mr.createJobConf();
+ m_conf.setInt("mapred.submit.replication", 2);
+ m_conf.set("dfs.datanode.address", "0.0.0.0:0");
+ m_conf.set("dfs.datanode.http.address", "0.0.0.0:0");
+ m_conf.write(new FileOutputStream(conf_file));
+
+ // Set the system properties needed by Pig
+ System.setProperty("cluster", m_conf.get("mapred.job.tracker"));
+ System.setProperty("namenode", m_conf.get("fs.default.name"));
+ System.setProperty("junit.hadoop.conf", conf_dir.getPath());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Returns the single instance of class MiniClusterBuilder that
+ * represents the resouces for a mini dfs cluster and a mini
+ * mapreduce cluster.
+ */
+ public static MiniCluster buildCluster() {
+ return INSTANCE;
+ }
+
+ protected void finalize() {
+ shutdownMiniDfsAndMrClusters();
+ }
+
+ private void shutdownMiniDfsAndMrClusters() {
+ try {
+ if (m_fileSys != null) { m_fileSys.close(); }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ if (m_dfs != null) { m_dfs.shutdown(); }
+ if (m_mr != null) { m_mr.shutdown(); }
+ }
+}
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestAlgebraicEval.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestAlgebraicEval.java?rev=631443&r1=631442&r2=631443&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestAlgebraicEval.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestAlgebraicEval.java Tue Feb 26 16:51:49 2008
@@ -34,10 +34,10 @@
public class TestAlgebraicEval extends TestCase {
- private String initString = "mapreduce";
-
+ private String initString = "mapreduce";
+ MiniCluster cluster = MiniCluster.buildCluster();
@Test
- public void testGroupCountWithMultipleFields() throws Exception {
+ public void testGroupCountWithMultipleFields() throws Throwable {
int LOOP_COUNT = 1024;
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
@@ -87,7 +87,7 @@
}
@Test
- public void testGroupCount() throws Exception {
+ public void testGroupCount() throws Throwable {
long LOOP_COUNT = 1024;
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
@@ -106,10 +106,8 @@
assertEquals(count.longValue(), LOOP_COUNT);
}
-
-
@Test
- public void testGroupReorderCount() throws Exception {
+ public void testGroupReorderCount() throws Throwable {
long LOOP_COUNT = 1024;
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
@@ -131,7 +129,7 @@
@Test
- public void testGroupUniqueColumnCount() throws Exception {
+ public void testGroupUniqueColumnCount() throws Throwable {
int LOOP_COUNT = 1024;
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
@@ -160,7 +158,7 @@
}
@Test
- public void testGroupDuplicateColumnCount() throws Exception {
+ public void testGroupDuplicateColumnCount() throws Throwable {
int LOOP_COUNT = 1024;
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestBuiltin.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestBuiltin.java?rev=631443&r1=631442&r2=631443&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestBuiltin.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestBuiltin.java Tue Feb 26 16:51:49 2008
@@ -18,9 +18,6 @@
package org.apache.pig.test;
import java.io.File;
-import java.io.FileOutputStream;
-import java.io.InputStream;
-import java.io.OutputStream;
import java.io.PrintWriter;
import java.util.HashMap;
import java.util.Iterator;
@@ -44,8 +41,8 @@
import org.apache.pig.impl.PigContext;
public class TestBuiltin extends TestCase {
-
- private String initString = "local";
+
+ private String initString = "local";
// Builtin MATH Functions
// =======================
@@ -288,6 +285,27 @@
Tuple f1 = p1.getNext();
assertTrue(f1.size() == arity1);
+ LoadFunc p15 = new PigStorage();
+ StringBuilder sb = new StringBuilder();
+ int LOOP_COUNT = 1024;
+ for (int i = 0; i < LOOP_COUNT; i++) {
+ for (int j = 0; j < LOOP_COUNT; j++) {
+ sb.append(i + "\t" + i + "\t" + j % 2 + "\n");
+ }
+ }
+ byte bytes[] = sb.toString().getBytes();
+ FakeFSInputStream ffis15 = new FakeFSInputStream(bytes);
+ p15.bindTo(null, new BufferedPositionedInputStream(ffis15), 0, bytes.length);
+ int count = 0;
+ while (true) {
+ Tuple f15 = p15.getNext();
+ if (f15 == null)
+ break;
+ count++;
+ assertEquals(3, f15.size());
+ }
+ assertEquals(LOOP_COUNT * LOOP_COUNT, count);
+
String input2 = ":this:has:a:leading:colon\n";
int arity2 = 6;
@@ -420,64 +438,64 @@
}
@Test
- public void testShellFuncSingle() throws Exception {
- //ShellBagEvalFunc func = new ShellBagEvalFunc("tr o 0");
- PigServer pig = new PigServer(initString);
-
- File tempFile = File.createTempFile("tmp", ".dat");
- PrintWriter writer = new PrintWriter(tempFile);
- writer.println("foo");
- writer.println("boo");
- writer.close();
-
- pig.registerFunction("myTr",ShellBagEvalFunc.class.getName() + "('tr o 0')");
- pig.registerQuery("a = load 'file:" + tempFile + "';");
- pig.registerQuery("b = foreach a generate myTr(*);");
- Iterator<Tuple> iter = pig.openIterator("b");
-
- Tuple t;
-
- assertTrue(iter.hasNext());
- t = iter.next();
- assertEquals("{(f00)}", t.get(0).toString());
- assertTrue(iter.hasNext());
- t = iter.next();
- assertEquals("{(b00)}", t.get(0).toString());
- assertFalse(iter.hasNext());
- tempFile.delete();
+ public void testShellFuncSingle() throws Throwable {
+ //ShellBagEvalFunc func = new ShellBagEvalFunc("tr o 0");
+ PigServer pig = new PigServer(initString);
+
+ File tempFile = File.createTempFile("tmp", ".dat");
+ PrintWriter writer = new PrintWriter(tempFile);
+ writer.println("foo");
+ writer.println("boo");
+ writer.close();
+
+ pig.registerFunction("myTr",ShellBagEvalFunc.class.getName() + "('tr o 0')");
+ pig.registerQuery("a = load 'file:" + tempFile + "';");
+ pig.registerQuery("b = foreach a generate myTr(*);");
+ Iterator<Tuple> iter = pig.openIterator("b");
+
+ Tuple t;
+
+ assertTrue(iter.hasNext());
+ t = iter.next();
+ assertEquals("{(f00)}", t.get(0).toString());
+ assertTrue(iter.hasNext());
+ t = iter.next();
+ assertEquals("{(b00)}", t.get(0).toString());
+ assertFalse(iter.hasNext());
+ tempFile.delete();
}
@Test
- public void testShellFuncMultiple() throws Exception {
+ public void testShellFuncMultiple() throws Throwable {
- PigServer pig = new PigServer(initString);
- final int numTimes = 100;
-
- File tempFile = File.createTempFile("tmp", ".dat");
- PrintWriter writer = new PrintWriter(tempFile);
- for (int i=0; i< numTimes; i++){
- writer.println(i+"oo");
- }
- writer.close();
-
- pig.registerFunction("tr1",ShellBagEvalFunc.class.getName() + "('tr o A')");
- pig.registerFunction("tr2",ShellBagEvalFunc.class.getName() + "('tr o B')");
- pig.registerQuery("a = load 'file:" + tempFile + "';");
- pig.registerQuery("b = foreach a generate tr1(*),tr2(*);");
- Iterator<Tuple> iter = pig.openIterator("b");
-
- for (int i=0; i< numTimes; i++){
- Tuple t = iter.next();
+ PigServer pig = new PigServer(initString);
+ final int numTimes = 100;
+
+ File tempFile = File.createTempFile("tmp", ".dat");
+ PrintWriter writer = new PrintWriter(tempFile);
+ for (int i=0; i< numTimes; i++){
+ writer.println(i+"oo");
+ }
+ writer.close();
+
+ pig.registerFunction("tr1",ShellBagEvalFunc.class.getName() + "('tr o A')");
+ pig.registerFunction("tr2",ShellBagEvalFunc.class.getName() + "('tr o B')");
+ pig.registerQuery("a = load 'file:" + tempFile + "';");
+ pig.registerQuery("b = foreach a generate tr1(*),tr2(*);");
+ Iterator<Tuple> iter = pig.openIterator("b");
+
+ for (int i=0; i< numTimes; i++){
+ Tuple t = iter.next();
DataBag b = DataType.toBag(t.get(0));
Tuple t1 = b.iterator().next();
- assertEquals(i+"AA", t1.get(0).toString());
+ assertEquals(i+"AA", t1.get(0).toString());
b = DataType.toBag(t.get(1));
t1 = b.iterator().next();
- assertEquals(i+"BB", t1.get(0).toString());
- }
-
- assertFalse(iter.hasNext());
- tempFile.delete();
+ assertEquals(i+"BB", t1.get(0).toString());
+ }
+
+ assertFalse(iter.hasNext());
+ tempFile.delete();
}
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestCmdLineParser.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestCmdLineParser.java?rev=631443&r1=631442&r2=631443&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestCmdLineParser.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestCmdLineParser.java Tue Feb 26 16:51:49 2008
@@ -32,251 +32,251 @@
@Test
public void testRegisterDash()
{
- String[] args = {"a", "b", "c"};
- CmdLineParser p = new CmdLineParser(args);
- try {
- p.registerOpt('-', "alpha", CmdLineParser.ValueExpected.NOT_ACCEPTED);
- fail("Should have thrown an AssertionError");
- } catch (AssertionError e) {
- assertEquals(e.getMessage(),
- "CmdLineParser: '-' is not a legal single character designator.");
- }
+ String[] args = {"a", "b", "c"};
+ CmdLineParser p = new CmdLineParser(args);
+ try {
+ p.registerOpt('-', "alpha", CmdLineParser.ValueExpected.NOT_ACCEPTED);
+ fail("Should have thrown an AssertionError");
+ } catch (AssertionError e) {
+ assertEquals(e.getMessage(),
+ "CmdLineParser: '-' is not a legal single character designator.");
+ }
}
@Test
public void testDoubleRegisterShort()
{
- String[] args = {"a", "b", "c"};
- CmdLineParser p = new CmdLineParser(args);
- try {
- p.registerOpt('a', "alpha", CmdLineParser.ValueExpected.NOT_ACCEPTED);
- p.registerOpt('a', "beta", CmdLineParser.ValueExpected.NOT_ACCEPTED);
- fail("Should have thrown an AssertionError");
- } catch (AssertionError e) {
- assertEquals(e.getMessage(),
- "CmdLineParser: You have already registered option a");
- }
+ String[] args = {"a", "b", "c"};
+ CmdLineParser p = new CmdLineParser(args);
+ try {
+ p.registerOpt('a', "alpha", CmdLineParser.ValueExpected.NOT_ACCEPTED);
+ p.registerOpt('a', "beta", CmdLineParser.ValueExpected.NOT_ACCEPTED);
+ fail("Should have thrown an AssertionError");
+ } catch (AssertionError e) {
+ assertEquals(e.getMessage(),
+ "CmdLineParser: You have already registered option a");
+ }
}
@Test
public void testDoubleRegisterLong()
{
- String[] args = {"a", "b", "c"};
- CmdLineParser p = new CmdLineParser(args);
- try {
- p.registerOpt('a', "alpha", CmdLineParser.ValueExpected.NOT_ACCEPTED);
- p.registerOpt('b', "alpha", CmdLineParser.ValueExpected.NOT_ACCEPTED);
- fail("Should have thrown an AssertionError");
- } catch (AssertionError e) {
- assertEquals(e.getMessage(),
- "CmdLineParser: You have already registered option alpha");
- }
+ String[] args = {"a", "b", "c"};
+ CmdLineParser p = new CmdLineParser(args);
+ try {
+ p.registerOpt('a', "alpha", CmdLineParser.ValueExpected.NOT_ACCEPTED);
+ p.registerOpt('b', "alpha", CmdLineParser.ValueExpected.NOT_ACCEPTED);
+ fail("Should have thrown an AssertionError");
+ } catch (AssertionError e) {
+ assertEquals(e.getMessage(),
+ "CmdLineParser: You have already registered option alpha");
+ }
}
@Test
public void testRegister()
{
- String[] args = {"a", "b", "c"};
- CmdLineParser p = new CmdLineParser(args);
- p.registerOpt('a', "alpha", CmdLineParser.ValueExpected.NOT_ACCEPTED);
- p.registerOpt('b', "beta", CmdLineParser.ValueExpected.REQUIRED);
- p.registerOpt('c', null, CmdLineParser.ValueExpected.OPTIONAL);
+ String[] args = {"a", "b", "c"};
+ CmdLineParser p = new CmdLineParser(args);
+ p.registerOpt('a', "alpha", CmdLineParser.ValueExpected.NOT_ACCEPTED);
+ p.registerOpt('b', "beta", CmdLineParser.ValueExpected.REQUIRED);
+ p.registerOpt('c', null, CmdLineParser.ValueExpected.OPTIONAL);
}
@Test
public void testParseNoArgs() throws ParseException
{
- String[] args = {};
- CmdLineParser p = new CmdLineParser(args);
- p.registerOpt('a', "alpha", CmdLineParser.ValueExpected.NOT_ACCEPTED);
- assertEquals(p.getNextOpt(), CmdLineParser.EndOfOpts);
+ String[] args = {};
+ CmdLineParser p = new CmdLineParser(args);
+ p.registerOpt('a', "alpha", CmdLineParser.ValueExpected.NOT_ACCEPTED);
+ assertEquals(p.getNextOpt(), CmdLineParser.EndOfOpts);
}
@Test
public void testParseNoDash() throws ParseException
{
- String[] args = {"a"};
- CmdLineParser p = new CmdLineParser(args);
- p.registerOpt('a', "alpha", CmdLineParser.ValueExpected.NOT_ACCEPTED);
- assertEquals(p.getNextOpt(), CmdLineParser.EndOfOpts);
- String[] remainders = p.getRemainingArgs();
- assertEquals(remainders[0], "a");
+ String[] args = {"a"};
+ CmdLineParser p = new CmdLineParser(args);
+ p.registerOpt('a', "alpha", CmdLineParser.ValueExpected.NOT_ACCEPTED);
+ assertEquals(p.getNextOpt(), CmdLineParser.EndOfOpts);
+ String[] remainders = p.getRemainingArgs();
+ assertEquals(remainders[0], "a");
}
@Test
public void testParseLongShortNoLeftover() throws ParseException
{
- String[] args = {"-a", "--beta", "beth", "--c"};
- CmdLineParser p = new CmdLineParser(args);
- p.registerOpt('a', "alpha", CmdLineParser.ValueExpected.NOT_ACCEPTED);
- p.registerOpt('b', "beta", CmdLineParser.ValueExpected.REQUIRED);
- p.registerOpt('c', null, CmdLineParser.ValueExpected.OPTIONAL);
- assertEquals(p.getNextOpt(), 'a');
- assertEquals(p.getNextOpt(), 'b');
- assertEquals(p.getValStr(), "beth");
- assertEquals(p.getNextOpt(), 'c');
- assertNull(p.getValStr());
- assertEquals(p.getNextOpt(), CmdLineParser.EndOfOpts);
- assertNull(p.getRemainingArgs());
+ String[] args = {"-a", "--beta", "beth", "--c"};
+ CmdLineParser p = new CmdLineParser(args);
+ p.registerOpt('a', "alpha", CmdLineParser.ValueExpected.NOT_ACCEPTED);
+ p.registerOpt('b', "beta", CmdLineParser.ValueExpected.REQUIRED);
+ p.registerOpt('c', null, CmdLineParser.ValueExpected.OPTIONAL);
+ assertEquals(p.getNextOpt(), 'a');
+ assertEquals(p.getNextOpt(), 'b');
+ assertEquals(p.getValStr(), "beth");
+ assertEquals(p.getNextOpt(), 'c');
+ assertNull(p.getValStr());
+ assertEquals(p.getNextOpt(), CmdLineParser.EndOfOpts);
+ assertNull(p.getRemainingArgs());
}
@Test
public void testParseLongShortLeftover1() throws ParseException
{
- String[] args = {"-a", "--beta", "beth", "--c", "gimel", "-", "hi", "i'm", "left",
- "over"};
- CmdLineParser p = new CmdLineParser(args);
- p.registerOpt('a', "alpha", CmdLineParser.ValueExpected.NOT_ACCEPTED);
- p.registerOpt('b', "beta", CmdLineParser.ValueExpected.REQUIRED);
- p.registerOpt('c', null, CmdLineParser.ValueExpected.OPTIONAL);
- assertEquals(p.getNextOpt(), 'a');
- assertEquals(p.getNextOpt(), 'b');
- assertEquals(p.getValStr(), "beth");
- assertEquals(p.getNextOpt(), 'c');
- assertEquals(p.getValStr(), "gimel");
- assertEquals(p.getNextOpt(), CmdLineParser.EndOfOpts);
- String[] r = p.getRemainingArgs();
- assertEquals(r.length, 4);
+ String[] args = {"-a", "--beta", "beth", "--c", "gimel", "-", "hi", "i'm", "left",
+ "over"};
+ CmdLineParser p = new CmdLineParser(args);
+ p.registerOpt('a', "alpha", CmdLineParser.ValueExpected.NOT_ACCEPTED);
+ p.registerOpt('b', "beta", CmdLineParser.ValueExpected.REQUIRED);
+ p.registerOpt('c', null, CmdLineParser.ValueExpected.OPTIONAL);
+ assertEquals(p.getNextOpt(), 'a');
+ assertEquals(p.getNextOpt(), 'b');
+ assertEquals(p.getValStr(), "beth");
+ assertEquals(p.getNextOpt(), 'c');
+ assertEquals(p.getValStr(), "gimel");
+ assertEquals(p.getNextOpt(), CmdLineParser.EndOfOpts);
+ String[] r = p.getRemainingArgs();
+ assertEquals(r.length, 4);
}
// has two dashes instead of one for end of args
@Test
public void testParseLongShortLeftover2() throws ParseException
{
- String[] args = {"-a", "-beta", "beth", "--c", "gimel", "--", "hi", "i'm", "left",
- "over"};
- CmdLineParser p = new CmdLineParser(args);
- p.registerOpt('a', "alpha", CmdLineParser.ValueExpected.NOT_ACCEPTED);
- p.registerOpt('b', "beta", CmdLineParser.ValueExpected.REQUIRED);
- p.registerOpt('c', null, CmdLineParser.ValueExpected.OPTIONAL);
- assertEquals(p.getNextOpt(), 'a');
- assertEquals(p.getNextOpt(), 'b');
- assertEquals(p.getValStr(), "beth");
- assertEquals(p.getNextOpt(), 'c');
- assertEquals(p.getValStr(), "gimel");
- assertEquals(p.getNextOpt(), CmdLineParser.EndOfOpts);
- String[] r = p.getRemainingArgs();
- assertEquals(r.length, 4);
+ String[] args = {"-a", "-beta", "beth", "--c", "gimel", "--", "hi", "i'm", "left",
+ "over"};
+ CmdLineParser p = new CmdLineParser(args);
+ p.registerOpt('a', "alpha", CmdLineParser.ValueExpected.NOT_ACCEPTED);
+ p.registerOpt('b', "beta", CmdLineParser.ValueExpected.REQUIRED);
+ p.registerOpt('c', null, CmdLineParser.ValueExpected.OPTIONAL);
+ assertEquals(p.getNextOpt(), 'a');
+ assertEquals(p.getNextOpt(), 'b');
+ assertEquals(p.getValStr(), "beth");
+ assertEquals(p.getNextOpt(), 'c');
+ assertEquals(p.getValStr(), "gimel");
+ assertEquals(p.getNextOpt(), CmdLineParser.EndOfOpts);
+ String[] r = p.getRemainingArgs();
+ assertEquals(r.length, 4);
}
@Test
public void testParseLongShortLeftover3() throws ParseException
{
- String[] args = {"-a", "--beta", "5", "--c", "--"};
- CmdLineParser p = new CmdLineParser(args);
- p.registerOpt('a', "alpha", CmdLineParser.ValueExpected.NOT_ACCEPTED);
- p.registerOpt('b', "beta", CmdLineParser.ValueExpected.REQUIRED);
- p.registerOpt('c', null, CmdLineParser.ValueExpected.OPTIONAL);
- assertEquals(p.getNextOpt(), 'a');
- assertEquals(p.getNextOpt(), 'b');
- Integer ii = p.getValInt();
- assertEquals(ii.intValue(), 5);
- assertEquals(p.getNextOpt(), 'c');
- assertNull(p.getValInt());
- assertEquals(p.getNextOpt(), CmdLineParser.EndOfOpts);
- String[] r = p.getRemainingArgs();
- assertNull(p.getRemainingArgs());
+ String[] args = {"-a", "--beta", "5", "--c", "--"};
+ CmdLineParser p = new CmdLineParser(args);
+ p.registerOpt('a', "alpha", CmdLineParser.ValueExpected.NOT_ACCEPTED);
+ p.registerOpt('b', "beta", CmdLineParser.ValueExpected.REQUIRED);
+ p.registerOpt('c', null, CmdLineParser.ValueExpected.OPTIONAL);
+ assertEquals(p.getNextOpt(), 'a');
+ assertEquals(p.getNextOpt(), 'b');
+ Integer ii = p.getValInt();
+ assertEquals(ii.intValue(), 5);
+ assertEquals(p.getNextOpt(), 'c');
+ assertNull(p.getValInt());
+ assertEquals(p.getNextOpt(), CmdLineParser.EndOfOpts);
+ String[] r = p.getRemainingArgs();
+ assertNull(p.getRemainingArgs());
}
@Test
public void testParseValueNotAcceptedProvided1() throws ParseException
{
- String[] args = {"-a", "aleph"};
- CmdLineParser p = new CmdLineParser(args);
- p.registerOpt('a', "alpha", CmdLineParser.ValueExpected.NOT_ACCEPTED);
- assertEquals(p.getNextOpt(), 'a');
- String[] r = p.getRemainingArgs();
- assertEquals(r.length, 1);
- assertEquals(r[0], "aleph");
+ String[] args = {"-a", "aleph"};
+ CmdLineParser p = new CmdLineParser(args);
+ p.registerOpt('a', "alpha", CmdLineParser.ValueExpected.NOT_ACCEPTED);
+ assertEquals(p.getNextOpt(), 'a');
+ String[] r = p.getRemainingArgs();
+ assertEquals(r.length, 1);
+ assertEquals(r[0], "aleph");
}
@Test
public void testParseValueNotAcceptedProvided2() throws ParseException
{
- String[] args = {"-alpha", "aleph"};
- CmdLineParser p = new CmdLineParser(args);
- p.registerOpt('a', "alpha", CmdLineParser.ValueExpected.NOT_ACCEPTED);
- assertEquals(p.getNextOpt(), 'a');
- String[] r = p.getRemainingArgs();
- assertEquals(r.length, 1);
- assertEquals(r[0], "aleph");
+ String[] args = {"-alpha", "aleph"};
+ CmdLineParser p = new CmdLineParser(args);
+ p.registerOpt('a', "alpha", CmdLineParser.ValueExpected.NOT_ACCEPTED);
+ assertEquals(p.getNextOpt(), 'a');
+ String[] r = p.getRemainingArgs();
+ assertEquals(r.length, 1);
+ assertEquals(r[0], "aleph");
}
@Test
public void testParseValueRequiredNotProvided1()
{
- String[] args = {"-a"};
- CmdLineParser p = new CmdLineParser(args);
- p.registerOpt('a', "alpha", CmdLineParser.ValueExpected.REQUIRED);
- try {
- p.getNextOpt();
- fail("Should have thrown a ParseException");
- } catch (ParseException e) {
- assertEquals(e.getMessage(),
- "Option -a requires a value but you did not provide one.");
- }
+ String[] args = {"-a"};
+ CmdLineParser p = new CmdLineParser(args);
+ p.registerOpt('a', "alpha", CmdLineParser.ValueExpected.REQUIRED);
+ try {
+ p.getNextOpt();
+ fail("Should have thrown a ParseException");
+ } catch (ParseException e) {
+ assertEquals(e.getMessage(),
+ "Option -a requires a value but you did not provide one.");
+ }
}
@Test
public void testParseValueRequiredNotProvided2()
{
- String[] args = {"--alpha", "-b"};
- CmdLineParser p = new CmdLineParser(args);
- p.registerOpt('a', "alpha", CmdLineParser.ValueExpected.REQUIRED);
- p.registerOpt('b', "beta", CmdLineParser.ValueExpected.NOT_ACCEPTED);
- try {
- p.getNextOpt();
- fail("Should have thrown a ParseException");
- } catch (ParseException e) {
- assertEquals(e.getMessage(),
- "Option --alpha requires a value but you did not provide one.");
- }
+ String[] args = {"--alpha", "-b"};
+ CmdLineParser p = new CmdLineParser(args);
+ p.registerOpt('a', "alpha", CmdLineParser.ValueExpected.REQUIRED);
+ p.registerOpt('b', "beta", CmdLineParser.ValueExpected.NOT_ACCEPTED);
+ try {
+ p.getNextOpt();
+ fail("Should have thrown a ParseException");
+ } catch (ParseException e) {
+ assertEquals(e.getMessage(),
+ "Option --alpha requires a value but you did not provide one.");
+ }
}
@Test
public void testParseValueStrForInt() throws ParseException
{
- String[] args = {"-alpha", "b"};
- CmdLineParser p = new CmdLineParser(args);
- p.registerOpt('a', "alpha", CmdLineParser.ValueExpected.REQUIRED);
- try {
- p.getNextOpt();
- Integer ii = p.getValInt();
- fail("Should have thrown a NumberFormatException");
- } catch (NumberFormatException e) {
- }
+ String[] args = {"-alpha", "b"};
+ CmdLineParser p = new CmdLineParser(args);
+ p.registerOpt('a', "alpha", CmdLineParser.ValueExpected.REQUIRED);
+ try {
+ p.getNextOpt();
+ Integer ii = p.getValInt();
+ fail("Should have thrown a NumberFormatException");
+ } catch (NumberFormatException e) {
+ }
}
@Test
public void testParseUnknownShort()
{
- String[] args = {"-alpha", "b", "-z"};
- CmdLineParser p = new CmdLineParser(args);
- p.registerOpt('a', "alpha", CmdLineParser.ValueExpected.REQUIRED);
- try {
- p.getNextOpt();
- assertEquals(p.getValStr(), "b");
- p.getNextOpt();
- fail("Should have thrown a ParseException");
- } catch (ParseException e) {
- assertEquals(e.getMessage(),
- "Found unknown option (-z) at position 3");
- }
+ String[] args = {"-alpha", "b", "-z"};
+ CmdLineParser p = new CmdLineParser(args);
+ p.registerOpt('a', "alpha", CmdLineParser.ValueExpected.REQUIRED);
+ try {
+ p.getNextOpt();
+ assertEquals(p.getValStr(), "b");
+ p.getNextOpt();
+ fail("Should have thrown a ParseException");
+ } catch (ParseException e) {
+ assertEquals(e.getMessage(),
+ "Found unknown option (-z) at position 3");
+ }
}
@Test
public void testParseUnknownLong()
{
- String[] args = {"--zeta"};
- CmdLineParser p = new CmdLineParser(args);
- p.registerOpt('a', "alpha", CmdLineParser.ValueExpected.REQUIRED);
- try {
- p.getNextOpt();
- fail("Should have thrown a ParseException");
- } catch (ParseException e) {
- assertEquals(e.getMessage(),
- "Found unknown option (--zeta) at position 1");
- }
+ String[] args = {"--zeta"};
+ CmdLineParser p = new CmdLineParser(args);
+ p.registerOpt('a', "alpha", CmdLineParser.ValueExpected.REQUIRED);
+ try {
+ p.getNextOpt();
+ fail("Should have thrown a ParseException");
+ } catch (ParseException e) {
+ assertEquals(e.getMessage(),
+ "Found unknown option (--zeta) at position 1");
+ }
}
}
Added: incubator/pig/branches/types/test/org/apache/pig/test/TestCombiner.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestCombiner.java?rev=631443&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestCombiner.java (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestCombiner.java Tue Feb 26 16:51:49 2008
@@ -0,0 +1,63 @@
+package org.apache.pig.test;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.junit.Test;
+import junit.framework.TestCase;
+
+import org.apache.pig.PigServer;
+import org.apache.pig.builtin.PigStorage;
+import org.apache.pig.data.Tuple;
+
+public class TestCombiner extends TestCase {
+
+ @Test
+ public void testLocal() throws Exception {
+ // run the test locally
+ runTest(new PigServer("local"));
+ }
+
+ @Test
+ public void testOnCluster() throws Exception {
+ // run the test on cluster
+ MiniCluster.buildCluster();
+ runTest(new PigServer("mapreduce"));
+
+ }
+
+ private void runTest(PigServer pig) throws IOException {
+ List<String> inputLines = new ArrayList<String>();
+ inputLines.add("a,b,1");
+ inputLines.add("a,b,1");
+ inputLines.add("a,c,1");
+ loadWithTestLoadFunc("A", pig, inputLines);
+
+ pig.registerQuery("B = group A by ($0, $1);");
+ pig.registerQuery("C = foreach B generate flatten(group), COUNT($1);");
+ Iterator<Tuple> resultIterator = pig.openIterator("C");
+ Tuple tuple = resultIterator.next();
+ assertEquals("(a, b, 2)", tuple.toString());
+ tuple = resultIterator.next();
+ assertEquals("(a, c, 1)", tuple.toString());
+ }
+
+ private void loadWithTestLoadFunc(String loadAlias, PigServer pig,
+ List<String> inputLines) throws IOException {
+ File inputFile = File.createTempFile("test", "txt");
+ PrintStream ps = new PrintStream(new FileOutputStream(inputFile));
+ for (String line : inputLines) {
+ ps.println(line);
+ }
+ ps.close();
+ pig.registerQuery(loadAlias + " = load 'file:"
+ + inputFile + "' using "
+ + PigStorage.class.getName() + "(',');");
+ }
+
+}
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestCompressedFiles.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestCompressedFiles.java?rev=631443&r1=631442&r2=631443&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestCompressedFiles.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestCompressedFiles.java Tue Feb 26 16:51:49 2008
@@ -26,15 +26,21 @@
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.pig.PigServer;
import org.apache.pig.builtin.DIFF;
import junit.framework.TestCase;
public class TestCompressedFiles extends TestCase {
+
+ private final Log log = LogFactory.getLog(getClass());
+ MiniCluster cluster = MiniCluster.buildCluster();
+
File datFile;
File gzFile;
@Override
- @Before
+ @Before
protected void setUp() throws Exception {
datFile = File.createTempFile("compTest", ".dat");
gzFile = File.createTempFile("compTest", ".gz");
@@ -58,21 +64,21 @@
}
@Override
- @After
+ @After
protected void tearDown() throws Exception {
datFile.delete();
gzFile.delete();
}
@Test
- public void testCompressed1() throws Exception {
+ public void testCompressed1() throws Throwable {
PigServer pig = new PigServer("mapreduce");
pig.registerQuery("A = foreach (cogroup (load 'file:"+gzFile+"') by $1, (load 'file:"+datFile + "') by $1) generate flatten( " + DIFF.class.getName() + "($1.$1,$2.$1)) ;");
Iterator it = pig.openIterator("A");
boolean success = true;
while(it.hasNext()) {
success = false;
- System.out.println(it.next());
+ log.info(it.next());
}
assertTrue(success);
}
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestEvalPipeline.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestEvalPipeline.java?rev=631443&r1=631442&r2=631443&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestEvalPipeline.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestEvalPipeline.java Tue Feb 26 16:51:49 2008
@@ -46,6 +46,7 @@
public class TestEvalPipeline extends TestCase {
String initString = "mapreduce";
+ MiniCluster cluster = MiniCluster.buildCluster();
TupleFactory mTf = TupleFactory.getInstance();
@@ -136,7 +137,7 @@
@Test
- public void testMapLookup() throws IOException{
+ public void testMapLookup() throws Exception {
PigServer pigServer = new PigServer(initString);
DataBag b = BagFactory.getInstance().newDefaultBag();
Map<Object, Object> colors = new HashMap<Object, Object>();
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestFilterOpNumeric.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestFilterOpNumeric.java?rev=631443&r1=631442&r2=631443&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestFilterOpNumeric.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestFilterOpNumeric.java Tue Feb 26 16:51:49 2008
@@ -24,22 +24,24 @@
import org.junit.Test;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.pig.PigServer;
import org.apache.pig.builtin.PigStorage;
import org.apache.pig.data.Tuple;
-import org.apache.pig.impl.util.PigLogger;
import junit.framework.TestCase;
public class TestFilterOpNumeric extends TestCase {
-
+ private final Log log = LogFactory.getLog(getClass());
+
private static int LOOP_COUNT = 1024;
private String initString = "mapreduce";
+ MiniCluster cluster = MiniCluster.buildCluster();
@Test
- public void testNumericEq() throws Exception {
- PigLogger.setAppenderForJunit();
+ public void testNumericEq() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -53,7 +55,7 @@
ps.close();
pig.registerQuery("A=load 'file:" + tmpFile + "' using "+PigStorage.class.getName() +"(':');");
String query = "A = filter A by $0 == $1;";
- System.out.println(query);
+ log.info(query);
pig.registerQuery(query);
Iterator it = pig.openIterator("A");
tmpFile.delete();
@@ -70,8 +72,7 @@
}
@Test
- public void testNumericNeq() throws Exception {
- PigLogger.setAppenderForJunit();
+ public void testNumericNeq() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -85,7 +86,7 @@
ps.close();
pig.registerQuery("A=load 'file:" + tmpFile + "' using " + PigStorage.class.getName() + "(':');");
String query = "A = filter A by $0 != $1;";
- System.out.println(query);
+ log.info(query);
pig.registerQuery(query);
Iterator it = pig.openIterator("A");
tmpFile.delete();
@@ -98,8 +99,7 @@
}
@Test
- public void testNumericGt() throws Exception {
- PigLogger.setAppenderForJunit();
+ public void testNumericGt() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -114,7 +114,7 @@
pig.registerQuery("A=load 'file:" + tmpFile + "' using " + PigStorage.class.getName() + "(':');");
String query = "A = filter A by $0 > $1;";
- System.out.println(query);
+ log.info(query);
pig.registerQuery(query);
Iterator it = pig.openIterator("A");
tmpFile.delete();
@@ -127,8 +127,7 @@
}
@Test
- public void testBinCond() throws Exception {
- PigLogger.setAppenderForJunit();
+ public void testBinCond() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -138,7 +137,7 @@
ps.close();
pig.registerQuery("A=load 'file:" + tmpFile + "';");
String query = "A = foreach A generate ($1 >= '"+ LOOP_COUNT+"'-'10'?'1':'0');";
- System.out.println(query);
+ log.info(query);
pig.registerQuery(query);
Iterator it = pig.openIterator("A");
tmpFile.delete();
@@ -147,9 +146,9 @@
Tuple t = (Tuple)it.next();
Double first = Double.valueOf(t.get(0).toString());
if (first == 1)
- count++;
+ count++;
else
- assertTrue(first == 0);
+ assertTrue(first == 0);
}
assertEquals("expected count of 10", 10, count);
@@ -157,8 +156,7 @@
@Test
- public void testNestedBinCond() throws Exception {
- PigLogger.setAppenderForJunit();
+ public void testNestedBinCond() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -168,7 +166,7 @@
ps.close();
pig.registerQuery("A=load 'file:" + tmpFile + "';");
String query = "A = foreach A generate ($0 < '10'?($1 >= '5' ? '2': '1') : '0');";
- System.out.println(query);
+ log.info(query);
pig.registerQuery(query);
Iterator it = pig.openIterator("A");
tmpFile.delete();
@@ -177,7 +175,7 @@
Tuple t = (Tuple)it.next();
Double first = Double.valueOf(t.get(0).toString());
count+=first;
- assertTrue(first == 1 || first == 2 || first == 0);
+ assertTrue(first == 1 || first == 2 || first == 0);
}
assertEquals("expected count of 15", 15, count);
@@ -186,11 +184,10 @@
@Test
- public void testNumericLt() throws Exception {
- PigLogger.setAppenderForJunit();
- PigServer pig = new PigServer(initString);
- File tmpFile = File.createTempFile("test", "txt");
- PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
+ public void testNumericLt() throws Throwable {
+ PigServer pig = new PigServer(initString);
+ File tmpFile = File.createTempFile("test", "txt");
+ PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
for(int i = 0; i < LOOP_COUNT; i++) {
if(i % 5 == 0) {
ps.println(i + ":" + (double)i);
@@ -202,7 +199,7 @@
pig.registerQuery("A=load 'file:" + tmpFile + "' using " + PigStorage.class.getName() + "(':');");
String query = "A = filter A by $0 < $1;";
- System.out.println(query);
+ log.info(query);
pig.registerQuery(query);
Iterator it = pig.openIterator("A");
tmpFile.delete();
@@ -212,13 +209,12 @@
Double second = Double.valueOf(t.get(1).toString());
assertTrue(first.compareTo(second) < 0);
}
-
+
}
@Test
- public void testNumericGte() throws Exception {
- PigLogger.setAppenderForJunit();
+ public void testNumericGte() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -235,7 +231,7 @@
pig.registerQuery("A=load 'file:" + tmpFile + "' using " + PigStorage.class.getName() + "(':');");
String query = "A = filter A by $0 >= $1;";
- System.out.println(query);
+ log.info(query);
pig.registerQuery(query);
Iterator it = pig.openIterator("A");
tmpFile.delete();
@@ -248,8 +244,7 @@
}
@Test
- public void testNumericLte() throws Exception {
- PigLogger.setAppenderForJunit();
+ public void testNumericLte() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -266,7 +261,7 @@
pig.registerQuery("A=load 'file:" + tmpFile + "' using " + PigStorage.class.getName() + "(':');");
String query = "A = filter A by $0 <= $1;";
- System.out.println(query);
+ log.info(query);
pig.registerQuery(query);
Iterator it = pig.openIterator("A");
tmpFile.delete();
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestFilterOpString.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestFilterOpString.java?rev=631443&r1=631442&r2=631443&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestFilterOpString.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestFilterOpString.java Tue Feb 26 16:51:49 2008
@@ -24,21 +24,23 @@
import org.junit.Test;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.pig.PigServer;
import org.apache.pig.builtin.PigStorage;
import org.apache.pig.data.Tuple;
-import org.apache.pig.impl.util.PigLogger;
import junit.framework.TestCase;
public class TestFilterOpString extends TestCase {
+ private final Log log = LogFactory.getLog(getClass());
private static int LOOP_COUNT = 1024;
private String initString = "mapreduce";
+ MiniCluster cluster = MiniCluster.buildCluster();
@Test
- public void testStringEq() throws Exception {
- PigLogger.setAppenderForJunit();
+ public void testStringEq() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -53,7 +55,7 @@
pig.registerQuery("A=load 'file:" + tmpFile + "' using " + PigStorage.class.getName() + "(':');");
String query = "A = filter A by $0 eq $1;";
- System.out.println(query);
+ log.info(query);
pig.registerQuery(query);
Iterator it = pig.openIterator("A");
tmpFile.delete();
@@ -66,8 +68,7 @@
}
@Test
- public void testStringNeq() throws Exception {
- PigLogger.setAppenderForJunit();
+ public void testStringNeq() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -82,7 +83,7 @@
pig.registerQuery("A=load 'file:" + tmpFile + "' using " + PigStorage.class.getName() + "(':');");
String query = "A = filter A by $0 neq $1;";
- System.out.println(query);
+ log.info(query);
pig.registerQuery(query);
Iterator it = pig.openIterator("A");
tmpFile.delete();
@@ -95,8 +96,7 @@
}
@Test
- public void testStringGt() throws Exception {
- PigLogger.setAppenderForJunit();
+ public void testStringGt() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -111,7 +111,7 @@
pig.registerQuery("A=load 'file:" + tmpFile + "' using " + PigStorage.class.getName() + "(':');");
String query = "A = filter A by $0 gt $1;";
- System.out.println(query);
+ log.info(query);
pig.registerQuery(query);
Iterator it = pig.openIterator("A");
tmpFile.delete();
@@ -126,8 +126,7 @@
@Test
- public void testStringGte() throws Exception {
- PigLogger.setAppenderForJunit();
+ public void testStringGte() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -145,7 +144,7 @@
pig.registerQuery("A=load 'file:" + tmpFile + "' using " + PigStorage.class.getName() + "(':');");
String query = "A = filter A by $0 gte $1;";
- System.out.println(query);
+ log.info(query);
pig.registerQuery(query);
Iterator it = pig.openIterator("A");
tmpFile.delete();
@@ -158,8 +157,7 @@
}
@Test
- public void testStringLt() throws Exception {
- PigLogger.setAppenderForJunit();
+ public void testStringLt() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -175,7 +173,7 @@
pig.registerQuery("A=load 'file:" + tmpFile + "' using " + PigStorage.class.getName() + "(':');");
String query = "A = filter A by $0 lt $1;";
- System.out.println(query);
+ log.info(query);
pig.registerQuery(query);
Iterator it = pig.openIterator("A");
tmpFile.delete();
@@ -188,8 +186,7 @@
}
@Test
- public void testStringLte() throws Exception {
- PigLogger.setAppenderForJunit();
+ public void testStringLte() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -207,7 +204,7 @@
pig.registerQuery("A=load 'file:" + tmpFile + "' using " + PigStorage.class.getName() + "(':');");
String query = "A = filter A by $0 lte $1;";
- System.out.println(query);
+ log.info(query);
pig.registerQuery(query);
Iterator it = pig.openIterator("A");
tmpFile.delete();
Added: incubator/pig/branches/types/test/org/apache/pig/test/TestGrunt.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestGrunt.java?rev=631443&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestGrunt.java (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestGrunt.java Tue Feb 26 16:51:49 2008
@@ -0,0 +1,32 @@
+package org.apache.pig.test;
+
+import org.junit.Test;
+import junit.framework.TestCase;
+
+import org.apache.pig.PigServer;
+import org.apache.pig.impl.PigContext;
+import org.apache.pig.tools.grunt.Grunt;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStreamReader;
+import java.io.BufferedReader;
+
+public class TestGrunt extends TestCase {
+ MiniCluster cluster = MiniCluster.buildCluster();
+
+
+ @Test
+ public void testCopyFromLocal() throws Throwable {
+ PigServer server = new PigServer("MAPREDUCE");
+ PigContext context = server.getPigContext();
+
+ String strCmd = "copyFromLocal /tmp/TestMe;";
+
+ ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
+ InputStreamReader reader = new InputStreamReader(cmd);
+
+ Grunt grunt = new Grunt(new BufferedReader(reader), context);
+
+ grunt.exec();
+ }
+}
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestInfixArithmetic.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestInfixArithmetic.java?rev=631443&r1=631442&r2=631443&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestInfixArithmetic.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestInfixArithmetic.java Tue Feb 26 16:51:49 2008
@@ -24,6 +24,8 @@
import org.junit.Test;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.pig.PigServer;
import org.apache.pig.builtin.PigStorage;
import org.apache.pig.data.DataType;
@@ -33,11 +35,14 @@
public class TestInfixArithmetic extends TestCase {
+ private final Log log = LogFactory.getLog(getClass());
+
private static int LOOP_COUNT = 1024;
private String initString = "mapreduce";
+ MiniCluster cluster = MiniCluster.buildCluster();
@Test
- public void testAdd() throws Exception {
+ public void testAdd() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -46,7 +51,7 @@
}
ps.close();
String query = "A = foreach (load 'file:" + tmpFile + "' using " + PigStorage.class.getName() + "(':')) generate $0, $0 + $1, $1;";
- System.out.println(query);
+ log.info(query);
pig.registerQuery(query);
Iterator it = pig.openIterator("A");
tmpFile.delete();
@@ -59,7 +64,7 @@
}
@Test
- public void testSubtract() throws Exception {
+ public void testSubtract() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -68,7 +73,7 @@
}
ps.close();
String query = "A = foreach (load 'file:" + tmpFile + "' using " + PigStorage.class.getName() + "(':')) generate $0, $0 - $1, $1 ;";
- System.out.println(query);
+ log.info(query);
pig.registerQuery(query);
Iterator it = pig.openIterator("A");
tmpFile.delete();
@@ -80,7 +85,7 @@
}
@Test
- public void testMultiply() throws Exception {
+ public void testMultiply() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -89,7 +94,7 @@
}
ps.close();
String query = "A = foreach (load 'file:" + tmpFile + "' using " + PigStorage.class.getName() + "(':')) generate $0, $0 * $1, $1 ;";
- System.out.println(query);
+ log.info(query);
pig.registerQuery(query);
Iterator it = pig.openIterator("A");
tmpFile.delete();
@@ -102,7 +107,7 @@
}
@Test
- public void testDivide() throws Exception {
+ public void testDivide() throws Throwable {
PigServer pig = new PigServer(initString);
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -111,7 +116,7 @@
}
ps.close();
String query = "A = foreach (load 'file:" + tmpFile + "' using " + PigStorage.class.getName() + "(':')) generate $0, $0 / $1, $1;";
- System.out.println(query);
+ log.info(query);
pig.registerQuery(query);
Iterator it = pig.openIterator("A");
tmpFile.delete();
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestLargeFile.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestLargeFile.java?rev=631443&r1=631442&r2=631443&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestLargeFile.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestLargeFile.java Tue Feb 26 16:51:49 2008
@@ -31,35 +31,39 @@
import junit.framework.TestCase;
import java.io.File;
+import java.io.IOException;
import java.io.FileOutputStream;
import java.util.Iterator;
import java.util.Random;
+import org.apache.pig.backend.executionengine.ExecException;
+
import org.apache.hadoop.conf.Configuration;
//This class tests pig behavior with large file spanning multiple blocks along with group and count functions
//Order and Distinct functions are also tested.
//This test would take a long time because of the large test files.
public class TestLargeFile extends TestCase {
-
- File datFile;
-
- private long defaultBlockSize = (new Configuration()).getLong("dfs.block.size", 0);
-
- private long total = defaultBlockSize >> 1;
- private int max_rand = 500;
- private String initString = "mapreduce";
-// private double sum = 0.0, sumIn = 0.0;
-
- Integer [] COUNT = new Integer[max_rand];
-
-
- PigServer pig;
- String fileName, tmpFile1;
-
- @Override
- @Before
- protected void setUp() throws Exception{
+
+ File datFile;
+
+ private long defaultBlockSize = (new Configuration()).getLong("dfs.block.size", 0);
+
+ private long total = defaultBlockSize >> 1;
+ private int max_rand = 500;
+ private String initString = "mapreduce";
+// private double sum = 0.0, sumIn = 0.0;
+ MiniCluster cluster = MiniCluster.buildCluster();
+
+ Integer [] COUNT = new Integer[max_rand];
+
+
+ PigServer pig;
+ String fileName, tmpFile1;
+
+ @Override
+ @Before
+ protected void setUp() throws Exception{
System.out.println("Generating test data...");
System.out.println("Default block size = " + defaultBlockSize);
@@ -72,7 +76,7 @@
Random rand = new Random();
for(int i = 0; i < max_rand; i++) {
- COUNT[i] = 0;
+ COUNT[i] = 0;
}
@@ -85,101 +89,108 @@
dat.close();
- pig = new PigServer(initString);
- fileName = "'" + FileLocalizer.hadoopify(datFile.toString(), pig.getPigContext()) + "'";
- tmpFile1 = "'" + FileLocalizer.getTemporaryPath(null, pig.getPigContext()).toString() + "'";
+ try {
+ pig = new PigServer(initString);
+ }
+ catch (ExecException e) {
+ IOException ioe = new IOException("Failed to create Pig server");
+ ioe.initCause(e);
+ throw ioe;
+ }
+ fileName = "'" + FileLocalizer.hadoopify(datFile.toString(), pig.getPigContext()) + "'";
+ tmpFile1 = "'" + FileLocalizer.getTemporaryPath(null, pig.getPigContext()).toString() + "'";
datFile.delete();
}
-
- @Override
- @After
- protected void tearDown() throws Exception {
-
-
- }
-
-
- @Test
- public void testLargeFile () throws Exception {
- System.out.println("Running testLargeFile...");
- pig.registerQuery("A = load " + fileName + ";");
- pig.registerQuery("A = group A by $0;");
- pig.store("A", tmpFile1, "BinStorage()");
-// pig.store("A", tmpFile1);
- pig.registerQuery("B = foreach A generate group, COUNT($1);");
-
- Iterator <Tuple> B = pig.openIterator("B");
-
- while(B.hasNext()) {
- Tuple temp = B.next();
- int index = DataType.toInteger(temp.get(0));
- int value = DataType.toInteger(temp.get(1));
- System.out.println("COUNT [" + index + "] = " + COUNT[index] + " B[" + index + "] = " + value);
-
- assertEquals(COUNT[index].intValue(), value);
-
- }
-
- }
-
- @Test
- public void testOrder () throws Exception {
- System.out.println("Running testOrder...");
- int N = 0, Nplus1 = 0;
- pig.registerQuery("A = load " + fileName + ";");
- pig.registerQuery("B = order A by $0;");
-
- Iterator <Tuple> B = pig.openIterator("B");
-
- if(B.hasNext()) {
- N = DataType.toInteger(B.next().get(0));
- }
-
- while(B.hasNext()) {
- int flag = 0;
- Nplus1 = DataType.toInteger(B.next().get(0));
- if(Nplus1 >= N) {
- flag = 1;
- }
- assertEquals(flag, 1);
-
- N = Nplus1;
-
- }
-
-
- }
-
- @Test
- public void testDistinct () throws Exception {
- System.out.println("Running testDistinct...");
- pig.registerQuery("A = load " + fileName + ";");
- pig.registerQuery("B = distinct A;");
-
- Iterator <Tuple> B = pig.openIterator("B");
-
- Integer [] COUNT_Test = new Integer [max_rand];
- Integer [] COUNT_Data = new Integer [max_rand];
-
- for(int i = 0; i < max_rand; i++) {
- COUNT_Test[i] = 0;
- if (COUNT[i] > 0) {
- COUNT_Data[i] = 1;
- } else {
- COUNT_Data[i] = 0;
- }
- }
-
- while(B.hasNext()) {
- int temp = DataType.toInteger(B.next().get(0));
- COUNT_Test[temp] ++;
- }
-
- for(int i = 0; i < max_rand; i++) {
- assertEquals(COUNT_Test[i].intValue(), COUNT_Data[i].intValue());
- }
-
- }
+
+ @Override
+ @After
+ protected void tearDown() throws Exception {
+
+
+ }
+
+
+ @Test
+ public void testLargeFile () throws Exception {
+ System.out.println("Running testLargeFile...");
+ pig.registerQuery("A = load " + fileName + ";");
+ pig.registerQuery("A = group A by $0;");
+ pig.store("A", tmpFile1, "BinStorage()");
+// pig.store("A", tmpFile1);
+ pig.registerQuery("B = foreach A generate group, COUNT($1);");
+
+ Iterator <Tuple> B = pig.openIterator("B");
+
+ while(B.hasNext()) {
+ Tuple temp = B.next();
+ int index = DataType.toInteger(temp.get(0));
+ int value = DataType.toInteger(temp.get(1));
+ System.out.println("COUNT [" + index + "] = " + COUNT[index] + " B[" + index + "] = " + value);
+
+ assertEquals(COUNT[index].intValue(), value);
+
+ }
+
+ }
+
+ @Test
+ public void testOrder () throws Exception {
+ System.out.println("Running testOrder...");
+ int N = 0, Nplus1 = 0;
+ pig.registerQuery("A = load " + fileName + ";");
+ pig.registerQuery("B = order A by $0;");
+
+ Iterator <Tuple> B = pig.openIterator("B");
+
+ if(B.hasNext()) {
+ N = DataType.toInteger(B.next().get(0));
+ }
+
+ while(B.hasNext()) {
+ int flag = 0;
+ Nplus1 = DataType.toInteger(B.next().get(0));
+ if(Nplus1 >= N) {
+ flag = 1;
+ }
+ assertEquals(flag, 1);
+
+ N = Nplus1;
+
+ }
+
+
+ }
+
+ @Test
+ public void testDistinct () throws Exception {
+ System.out.println("Running testDistinct...");
+ pig.registerQuery("A = load " + fileName + ";");
+ pig.registerQuery("B = distinct A;");
+
+ Iterator <Tuple> B = pig.openIterator("B");
+
+ Integer [] COUNT_Test = new Integer [max_rand];
+ Integer [] COUNT_Data = new Integer [max_rand];
+
+ for(int i = 0; i < max_rand; i++) {
+ COUNT_Test[i] = 0;
+ if (COUNT[i] > 0) {
+ COUNT_Data[i] = 1;
+ } else {
+ COUNT_Data[i] = 0;
+ }
+ }
+
+ while(B.hasNext()) {
+ int temp = DataType.toInteger(B.next().get(0));
+ COUNT_Test[temp] ++;
+ }
+
+ for(int i = 0; i < max_rand; i++) {
+ assertEquals(COUNT_Test[i].intValue(), COUNT_Data[i].intValue());
+ }
+
+ }
}
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java?rev=631443&r1=631442&r2=631443&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java Tue Feb 26 16:51:49 2008
@@ -26,6 +26,8 @@
import org.junit.Test;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.pig.LoadFunc;
import org.apache.pig.PigServer;
import org.apache.pig.builtin.PigStorage;
@@ -35,15 +37,18 @@
import org.apache.pig.PigServer.ExecType;
import org.apache.pig.impl.builtin.ShellBagEvalFunc;
import org.apache.pig.impl.io.BufferedPositionedInputStream;
+import org.apache.pig.impl.logicalLayer.OperatorKey;
import org.apache.pig.impl.logicalLayer.LOCogroup;
import org.apache.pig.impl.logicalLayer.LOEval;
import org.apache.pig.impl.logicalLayer.LogicalOperator;
import org.apache.pig.impl.logicalLayer.LogicalPlan;
import org.apache.pig.impl.logicalLayer.LogicalPlanBuilder;
-import org.apache.pig.impl.physicalLayer.IntermedResult;
+
public class TestLogicalPlanBuilder extends junit.framework.TestCase {
+ private final Log log = LogFactory.getLog(getClass());
+
@Test
public void testQuery1() {
String query = "foreach (load 'a') generate $1,$2;";
@@ -200,20 +205,22 @@
public void testQuery18() {
String query = "FOREACH (group (load 'a') ALL PARALLEL 16) generate group;";
LogicalPlan lp = buildPlan(query);
- LogicalOperator lo = lp.getRoot().getInputs().get(0);
+ Map<OperatorKey, LogicalOperator> logicalOpTable = lp.getOpTable();
+ OperatorKey logicalKey = lp.getRoot();
+ LogicalOperator lo = logicalOpTable.get(logicalOpTable.get(logicalKey).getInputs().get(0));
+
if (lo instanceof LOCogroup) {
assertTrue(((LOCogroup) lo).getRequestedParallelism() == 16);
} else {
fail("Error: Unexpected Parse Tree output");
}
-
}
@Test
public void testQuery19() {
buildPlan("a = load 'a';");
- buildPlan("a = filter a by $1 == '3';");
+ buildPlan("a = filter a by $1 == '3';");
}
@Test
@@ -224,54 +231,54 @@
@Test
public void testQuery21() {
- buildPlan("A = load 'a';");
- buildPlan("B = load 'b';");
+ buildPlan("A = load 'a';");
+ buildPlan("B = load 'b';");
buildPlan("foreach (cogroup A by ($1), B by ($1)) generate A, flatten(B.($1, $2, $3));");
}
@Test
public void testQuery22() {
- buildPlan("A = load 'a';");
- buildPlan("B = load 'b';");
- buildPlan("C = cogroup A by ($1), B by ($1);");
+ buildPlan("A = load 'a';");
+ buildPlan("B = load 'b';");
+ buildPlan("C = cogroup A by ($1), B by ($1);");
String query = "foreach C { " +
- "B = order B by $0; " +
- "generate FLATTEN(A), B.($1, $2, $3) ;" +
- "};" ;
+ "B = order B by $0; " +
+ "generate FLATTEN(A), B.($1, $2, $3) ;" +
+ "};" ;
buildPlan(query);
}
@Test
public void testQuery23() {
- buildPlan("A = load 'a';");
- buildPlan("B = load 'b';");
-
- buildPlan("C = cogroup A by ($1), B by ($1);");
+ buildPlan("A = load 'a';");
+ buildPlan("B = load 'b';");
- String query = "foreach C { " +
- "A = Distinct A; " +
- "B = FILTER A BY $1 < 'z'; " +
- "C = FILTER A BY $2 == $3;" +
- "B = ARRANGE B BY $1;" +
- "GENERATE A, FLATTEN(B.$0);" +
- "};";
+ buildPlan("C = cogroup A by ($1), B by ($1);");
+
+ String query = "foreach C { " +
+ "A = Distinct A; " +
+ "B = FILTER A BY $1 < 'z'; " +
+ "C = FILTER A BY $2 == $3;" +
+ "B = ARRANGE B BY $1;" +
+ "GENERATE A, FLATTEN(B.$0);" +
+ "};";
buildPlan(query);
}
@Test
public void testQuery24() {
- buildPlan("a = load 'a';");
-
- String query = "foreach a generate (($0 == $1) ? 'a' : $2), $4 ;";
+ buildPlan("a = load 'a';");
+
+ String query = "foreach a generate (($0 == $1) ? 'a' : $2), $4 ;";
buildPlan(query);
}
@Test
public void testQuery25() {
String query = "foreach (load 'a') {" +
- "B = FILTER $0 BY (($1 == $2) AND ('a' < 'b'));" +
- "generate B;" +
- "};";
+ "B = FILTER $0 BY (($1 == $2) AND ('a' < 'b'));" +
+ "generate B;" +
+ "};";
buildPlan(query);
}
@@ -284,9 +291,9 @@
@Test
public void testQuery27() {
String query = "foreach (load 'a'){" +
- "A = DISTINCT $3.$1;" +
- " generate " + TestApplyFunc.class.getName() + "($2, $1.($1, $4));" +
- "};";
+ "A = DISTINCT $3.$1;" +
+ " generate " + TestApplyFunc.class.getName() + "($2, $1.($1, $4));" +
+ "};";
buildPlan(query);
}
@@ -310,13 +317,13 @@
}
public static class TestStorageFunc implements LoadFunc{
- public void bindTo(String fileName, BufferedPositionedInputStream is, long offset, long end) throws IOException {
-
- }
-
- public Tuple getNext() throws IOException {
- return null;
- }
+ public void bindTo(String fileName, BufferedPositionedInputStream is, long offset, long end) throws IOException {
+
+ }
+
+ public Tuple getNext() throws IOException {
+ return null;
+ }
}
@Test
@@ -333,10 +340,10 @@
@Test
public void testQuery33() {
- buildPlan("A = load 'a' as (aCol1, aCol2);");
- buildPlan("B = load 'b' as (bCol1, bCol2);");
- buildPlan("C = cogroup A by (aCol1), B by bCol1;");
- String query = "foreach C generate group, A.aCol1;";
+ buildPlan("A = load 'a' as (aCol1, aCol2);");
+ buildPlan("B = load 'b' as (bCol1, bCol2);");
+ buildPlan("C = cogroup A by (aCol1), B by bCol1;");
+ String query = "foreach C generate group, A.aCol1;";
buildPlan(query);
}
@@ -344,10 +351,10 @@
@Test
//TODO: Nested schemas don't work now. Probably a bug in the new parser.
public void testQuery34() {
- buildPlan("A = load 'a' as (aCol1, aCol2 : (subCol1, subCol2));");
- buildPlan("A = filter A by aCol2 == '1';");
- buildPlan("B = load 'b' as (bCol1, bCol2);");
- String query = "foreach (cogroup A by (aCol1), B by bCol1 ) generate A.aCol2, B.bCol2 ;";
+ buildPlan("A = load 'a' as (aCol1, aCol2 : (subCol1, subCol2));");
+ buildPlan("A = filter A by aCol2 == '1';");
+ buildPlan("B = load 'b' as (bCol1, bCol2);");
+ String query = "foreach (cogroup A by (aCol1), B by bCol1 ) generate A.aCol2, B.bCol2 ;";
buildPlan(query);
}
@@ -369,28 +376,28 @@
public void testQueryFail37() {
String query = "A = load 'a'; asdasdas";
try{
- buildPlan(query);
+ buildPlan(query);
}catch(AssertionFailedError e){
- assertTrue(e.getMessage().contains("Exception"));
+ assertTrue(e.getMessage().contains("Exception"));
}
}
@Test
public void testQuery38(){
- String query = "c = cross (load 'a'), (load 'b');";
- buildPlan(query);
+ String query = "c = cross (load 'a'), (load 'b');";
+ buildPlan(query);
}
// @Test
// TODO: Schemas don't quite work yet
public void testQuery39(){
- buildPlan("a = load 'a' as (url, host, rank);");
- buildPlan("b = group a by (url,host); ");
- LogicalPlan lp = buildPlan("c = foreach b generate flatten(group.url), SUM(a.rank) as totalRank;");
- buildPlan("d = filter c by totalRank > '10';");
- buildPlan("e = foreach d generate url;");
-
+ buildPlan("a = load 'a' as (url, host, rank);");
+ buildPlan("b = group a by (url,host); ");
+ LogicalPlan lp = buildPlan("c = foreach b generate flatten(group.url), SUM(a.rank) as totalRank;");
+ buildPlan("d = filter c by totalRank > '10';");
+ buildPlan("e = foreach d generate url;");
+
}
@Test
@@ -399,128 +406,132 @@
buildPlan("a = FILTER (load 'a') BY (IsEmpty($2) AND ($3 == $2));");
}
+ @Test
public void testQuery41() {
- buildPlan("a = load 'a';");
- buildPlan("b = a as (host,url);");
- buildPlan("foreach b generate host;");
+ buildPlan("a = load 'a';");
+ buildPlan("b = a as (host,url);");
+ buildPlan("foreach b generate host;");
}
+ @Test
public void testQuery42() {
- buildPlan("a = load 'a';");
- buildPlan("b = foreach a generate $0 as url, $1 as rank;");
- buildPlan("foreach b generate url;");
+ buildPlan("a = load 'a';");
+ buildPlan("b = foreach a generate $0 as url, $1 as rank;");
+ buildPlan("foreach b generate url;");
}
+ @Test
public void testQuery43() {
- buildPlan("a = load 'a' as (url,hitCount);");
- buildPlan("b = load 'a' as (url,rank);");
- buildPlan("c = cogroup a by url, b by url;");
- buildPlan("d = foreach c generate group,flatten(a),flatten(b);");
- buildPlan("e = foreach d generate group, a::url, b::url, b::rank, rank;");
+ buildPlan("a = load 'a' as (url,hitCount);");
+ buildPlan("b = load 'a' as (url,rank);");
+ buildPlan("c = cogroup a by url, b by url;");
+ buildPlan("d = foreach c generate group,flatten(a),flatten(b);");
+ buildPlan("e = foreach d generate group, a::url, b::url, b::rank, rank;");
}
+ @Test
public void testQuery44() {
- buildPlan("a = load 'a' as (url, pagerank);");
- buildPlan("b = load 'b' as (url, query, rank);");
- buildPlan("c = cogroup a by (pagerank#'nonspam', url) , b by (rank/'2', url) ;");
- buildPlan("foreach c generate group.url;");
+ buildPlan("a = load 'a' as (url, pagerank);");
+ buildPlan("b = load 'b' as (url, query, rank);");
+ buildPlan("c = cogroup a by (pagerank#'nonspam', url) , b by (rank/'2', url) ;");
+ buildPlan("foreach c generate group.url;");
}
-
- public void testQueryFail44() {
+ @Test
+ public void testQueryFail44() throws Throwable {
PigServer pig = null;
try {
pig = new PigServer("local");
} catch (IOException e) {
assertTrue(false); // pig server failed for some reason
}
- pig.registerFunction("myTr",ShellBagEvalFunc.class.getName() + "('tr o 0')");
- try{
- pig.registerQuery("b = foreach (load 'a') generate myTr(myTr(*));");
+ pig.registerFunction("myTr",ShellBagEvalFunc.class.getName() + "('tr o 0')");
+ try{
+ pig.registerQuery("b = foreach (load 'a') generate myTr(myTr(*));");
}catch(Exception e){
- return;
+ return;
}
assertTrue(false);
}
- /*
+ /*
// Select
public void testQuery45() {
- buildPlan("A = load 'a' as (url,hitCount);");
- buildPlan("B = select url, hitCount from A;");
- buildPlan("C = select url, hitCount from B;");
+ buildPlan("A = load 'a' as (url,hitCount);");
+ buildPlan("B = select url, hitCount from A;");
+ buildPlan("C = select url, hitCount from B;");
}
//Select + Join
public void testQuery46() {
- buildPlan("A = load 'a' as (url,hitCount);");
- buildPlan("B = load 'b' as (url,pageRank);");
- buildPlan("C = select A.url, A.hitCount, B.pageRank from A join B on A.url == B.url;");
+ buildPlan("A = load 'a' as (url,hitCount);");
+ buildPlan("B = load 'b' as (url,pageRank);");
+ buildPlan("C = select A.url, A.hitCount, B.pageRank from A join B on A.url == B.url;");
}
// Mutliple Joins
public void testQuery47() {
- buildPlan("A = load 'a' as (url,hitCount);");
- buildPlan("B = load 'b' as (url,pageRank);");
- buildPlan("C = load 'c' as (pageRank, position);");
- buildPlan("B = select A.url, A.hitCount, B.pageRank from (A join B on A.url == B.url) join C on B.pageRank == C.pageRank;");
+ buildPlan("A = load 'a' as (url,hitCount);");
+ buildPlan("B = load 'b' as (url,pageRank);");
+ buildPlan("C = load 'c' as (pageRank, position);");
+ buildPlan("B = select A.url, A.hitCount, B.pageRank from (A join B on A.url == B.url) join C on B.pageRank == C.pageRank;");
}
// Group
public void testQuery48() {
- buildPlan("A = load 'a' as (url,hitCount);");
- buildPlan("C = select A.url, AVG(A.hitCount) from A group by url;");
+ buildPlan("A = load 'a' as (url,hitCount);");
+ buildPlan("C = select A.url, AVG(A.hitCount) from A group by url;");
}
// Join + Group
public void testQuery49() {
- buildPlan("A = load 'a' as (url,hitCount);");
- buildPlan("B = load 'b' as (url,pageRank);");
- buildPlan("C = select A.url, AVG(B.pageRank), SUM(A.hitCount) from A join B on A.url == B.url group by A.url;");
+ buildPlan("A = load 'a' as (url,hitCount);");
+ buildPlan("B = load 'b' as (url,pageRank);");
+ buildPlan("C = select A.url, AVG(B.pageRank), SUM(A.hitCount) from A join B on A.url == B.url group by A.url;");
}
// Group + Having
public void testQuery50() {
- buildPlan("A = load 'a' as (url,hitCount);");
- buildPlan("C = select A.url, AVG(A.hitCount) from A group by url having AVG(A.hitCount) > '6';");
+ buildPlan("A = load 'a' as (url,hitCount);");
+ buildPlan("C = select A.url, AVG(A.hitCount) from A group by url having AVG(A.hitCount) > '6';");
}
// Group + Having + Order
public void testQuery51() {
- buildPlan("A = load 'a' as (url,hitCount);");
- buildPlan("C = select A.url, AVG(A.hitCount) from A group by url order by A.url;");
+ buildPlan("A = load 'a' as (url,hitCount);");
+ buildPlan("C = select A.url, AVG(A.hitCount) from A group by url order by A.url;");
}
// Group + Having + Order
public void testQuery52() {
- buildPlan("A = load 'a' as (url,hitCount);");
- buildPlan("C = select A.url, AVG(A.hitCount) from A group by url having AVG(A.hitCount) > '6' order by A.url;");
+ buildPlan("A = load 'a' as (url,hitCount);");
+ buildPlan("C = select A.url, AVG(A.hitCount) from A group by url having AVG(A.hitCount) > '6' order by A.url;");
}
// Group + Having + Order 2
public void testQuery53() {
- buildPlan("A = load 'a' as (url,hitCount);");
- buildPlan("C = select A.url, AVG(A.hitCount) from A group by url having AVG(A.hitCount) > '6' order by AVG(A.hitCount);");
+ buildPlan("A = load 'a' as (url,hitCount);");
+ buildPlan("C = select A.url, AVG(A.hitCount) from A group by url having AVG(A.hitCount) > '6' order by AVG(A.hitCount);");
}
// Group + Having + Order 2
public void testQuery54() {
- buildPlan("A = load 'a' as (url,hitCount, size);");
- buildPlan("C = select A.url, AVG(A.hitCount) from A group by url having AVG(A.size) > '6' order by AVG(A.hitCount);");
+ buildPlan("A = load 'a' as (url,hitCount, size);");
+ buildPlan("C = select A.url, AVG(A.hitCount) from A group by url having AVG(A.size) > '6' order by AVG(A.hitCount);");
}
// Group + Having + Order 2
public void testQuery55() {
- buildPlan("A = load 'a' as (url,hitCount, size);");
- buildPlan("C = select A.url, AVG(A.hitCount), SUM(A.size) from A group by url having AVG(A.size) > '6' order by AVG(A.hitCount);");
+ buildPlan("A = load 'a' as (url,hitCount, size);");
+ buildPlan("C = select A.url, AVG(A.hitCount), SUM(A.size) from A group by url having AVG(A.size) > '6' order by AVG(A.hitCount);");
}
// Group + Having + Order 2
public void testQuery56() {
- buildPlan("A = load 'a' as (url,hitCount, date);");
- buildPlan("C = select A.url, A.date, SUM(A.hitCount) from A group by url, date having AVG(A.hitCount) > '6' order by A.date;");
+ buildPlan("A = load 'a' as (url,hitCount, date);");
+ buildPlan("C = select A.url, A.date, SUM(A.hitCount) from A group by url, date having AVG(A.hitCount) > '6' order by A.date;");
}
- */
+ */
// Helper Functions
// =================
@@ -534,27 +545,30 @@
LogicalPlanBuilder builder = new LogicalPlanBuilder(pigContext); //
try {
- LogicalPlan lp = builder.parse(query, queryResults);
- if (lp.getRoot() instanceof LOEval){
- System.out.println(query);
- System.out.println(((LOEval)lp.getRoot()).getSpec());
+ LogicalPlan lp = builder.parse("Test-Plan-Builder",
+ query,
+ aliases,
+ logicalOpTable);
+ if (logicalOpTable.get(lp.getRoot()) instanceof LOEval){
+ System.out.println(query);
+ System.out.println(((LOEval)logicalOpTable.get(lp.getRoot())).getSpec());
}
if (lp.getAlias()!=null){
- queryResults.put(lp.getAlias(), new IntermedResult(lp, pigContext));
+ aliases.put(lp.getAlias(), lp);
}
- //System.out.println(lp.root().outputSchema());
+
assertTrue(lp != null);
return lp;
} catch (IOException e) {
- // e.printStackTrace();
+ // log.error(e);
fail("IOException: " + e.getMessage());
} catch (Exception e) {
- e.printStackTrace();
+ log.error(e);
fail(e.getClass().getName() + ": " + e.getMessage() + " -- " + query);
}
return null;
}
- Map<String, IntermedResult> queryResults = new HashMap<String, IntermedResult>();
-
+ Map<String, LogicalPlan> aliases = new HashMap<String, LogicalPlan>();
+ Map<OperatorKey, LogicalOperator> logicalOpTable = new HashMap<OperatorKey, LogicalOperator>();
}