You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by dv...@apache.org on 2011/04/22 20:53:55 UTC
svn commit: r1095985 - in /pig/trunk: CHANGES.txt
test/org/apache/pig/test/TestTypedMap.java
Author: dvryaboy
Date: Fri Apr 22 18:53:54 2011
New Revision: 1095985
URL: http://svn.apache.org/viewvc?rev=1095985&view=rev
Log:
PIG-2011: Speed up TestTypedMap.java
Modified:
pig/trunk/CHANGES.txt
pig/trunk/test/org/apache/pig/test/TestTypedMap.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1095985&r1=1095984&r2=1095985&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Apr 22 18:53:54 2011
@@ -26,6 +26,8 @@ IMPROVEMENTS
OPTIMIZATIONS
+PIG-2011: Speed up TestTypedMap.java (dvryaboy)
+
BUG FIXES
Release 0.9.0 - Unreleased
Modified: pig/trunk/test/org/apache/pig/test/TestTypedMap.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestTypedMap.java?rev=1095985&r1=1095984&r2=1095985&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestTypedMap.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestTypedMap.java Fri Apr 22 18:53:54 2011
@@ -18,36 +18,24 @@
package org.apache.pig.test;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
import java.io.File;
-import java.io.FileWriter;
import java.io.IOException;
-import java.io.PrintWriter;
import java.util.Iterator;
-import java.util.List;
import java.util.Map;
import junit.framework.Assert;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.lib.FieldSelectionMapReduce;
-import org.apache.pig.EvalFunc;
import org.apache.pig.ExecType;
-import org.apache.pig.PigException;
import org.apache.pig.PigServer;
-import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.DataByteArray;
-import org.apache.pig.data.DataType;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.io.FileLocalizer;
-import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.parser.ParseException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
-import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
-import org.apache.pig.impl.util.LogUtils;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -56,44 +44,60 @@ import org.junit.Test;
public class TestTypedMap {
static MiniCluster cluster ;
+ private String tmpDirName = System.getProperty("java.io.tmpdir") + "/pigtest/tmp";
@Before
public void setUp() throws Exception {
FileLocalizer.setInitialized(false);
- }
+ File tmpDir = new File(tmpDirName);
+ if (tmpDir.exists()) {
+ for (File tmpFile : tmpDir.listFiles()) {
+ tmpFile.delete();
+ }
+ } else {
+ tmpDir.mkdirs();
+ }
+ }
@After
public void tearDown() throws Exception {
+ File tmpDir = new File(tmpDirName);
+ if (tmpDir.exists()) {
+ for (File tmpFile : tmpDir.listFiles()) {
+ tmpFile.delete();
+ }
+ tmpDir.delete();
+ }
}
-
+
@BeforeClass
public static void oneTimeSetup() throws IOException, Exception {
- cluster = MiniCluster.buildCluster();
+ cluster = MiniCluster.buildCluster();
}
-
+
@AfterClass
public static void oneTimeTearDown() throws Exception {
cluster.shutDown();
}
-
+
@Test
public void testSimpleLoad() throws IOException, ParseException {
- PigServer pig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer pig = new PigServer(ExecType.LOCAL, cluster.getProperties());
String[] input = {
"[key#1,key2#2]",
"[key#2]",
};
-
- Util.createInputFile(cluster, "table_testSimpleLoad", input);
-
+
+ Util.createInputFile(FileSystem.getLocal(new Configuration()), tmpDirName + "/table_testSimpleLoad", input);
+
String query =
- " a = load 'table_testSimpleLoad' as (m:map[int]);";
+ " a = load '" + tmpDirName + "/table_testSimpleLoad' as (m:map[int]);";
Util.registerMultiLineQuery(pig, query);
Schema sch = pig.dumpSchema("a");
assertEquals("Checking expected schema",sch.toString(), "{m: map[int]}");
Iterator<Tuple> it = pig.openIterator("a");
-
+
Assert.assertTrue(it.hasNext());
Tuple t = it.next();
Assert.assertTrue(t.size()==1);
@@ -104,65 +108,65 @@ public class TestTypedMap {
Assert.assertTrue(((Map)t.get(0)).get("key").toString().equals("1"));
Assert.assertTrue(((Map)t.get(0)).get("key2") instanceof Integer);
Assert.assertTrue(((Map)t.get(0)).get("key2").toString().equals("2"));
-
+
Assert.assertTrue(it.hasNext());
t = it.next();
Assert.assertTrue(((Map)t.get(0)).containsKey("key"));
Assert.assertTrue(((Map)t.get(0)).get("key") instanceof Integer);
Assert.assertTrue(((Map)t.get(0)).get("key").toString().equals("2"));
-
+
Assert.assertFalse(it.hasNext());
}
-
+
@Test
public void testSimpleMapKeyLookup() throws IOException, ParseException {
- PigServer pig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer pig = new PigServer(ExecType.LOCAL, cluster.getProperties());
String[] input = {
"[key#1,key2#2]",
"[key#2]",
};
-
- Util.createInputFile(cluster, "testSimpleMapKeyLookup", input);
-
+
+ Util.createInputFile(FileSystem.getLocal(new Configuration()), tmpDirName + "/testSimpleMapKeyLookup", input);
+
String query =
- "a = load 'testSimpleMapKeyLookup' as (m:map[int]);" +
+ "a = load '" + tmpDirName + "/testSimpleMapKeyLookup' as (m:map[int]);" +
"b = foreach a generate m#'key';";
Util.registerMultiLineQuery(pig, query);
Schema sch = pig.dumpSchema("b");
assertEquals("Checking expected schema",sch.toString(), "{int}");
Iterator<Tuple> it = pig.openIterator("b");
-
+
Assert.assertTrue(it.hasNext());
Tuple t = it.next();
Assert.assertTrue(t.size()==1);
Assert.assertTrue((Integer)t.get(0)==1);
-
+
Assert.assertTrue(it.hasNext());
t = it.next();
Assert.assertTrue(t.size()==1);
Assert.assertTrue((Integer)t.get(0)==2);
-
+
Assert.assertFalse(it.hasNext());
}
-
+
@Test
public void testSimpleMapCast() throws IOException, ParseException {
- PigServer pig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer pig = new PigServer(ExecType.LOCAL, cluster.getProperties());
String[] input = {
"[key#1,key2#2]",
"[key#2]",
};
-
- Util.createInputFile(cluster, "testSimpleMapCast", input);
-
+
+ Util.createInputFile(FileSystem.getLocal(new Configuration()), tmpDirName + "/testSimpleMapCast", input);
+
String query =
- "a = load 'testSimpleMapCast' as (m);" +
+ "a = load '" + tmpDirName + "/testSimpleMapCast' as (m);" +
"b = foreach a generate ([int])m;";
Util.registerMultiLineQuery(pig, query);
Schema sch = pig.dumpSchema("b");
assertEquals("Checking expected schema",sch.toString(), "{m: map[int]}");
Iterator<Tuple> it = pig.openIterator("b");
-
+
Assert.assertTrue(it.hasNext());
Tuple t = it.next();
Assert.assertTrue(t.size()==1);
@@ -173,32 +177,32 @@ public class TestTypedMap {
Assert.assertTrue(((Map)t.get(0)).get("key").toString().equals("1"));
Assert.assertTrue(((Map)t.get(0)).get("key2") instanceof Integer);
Assert.assertTrue(((Map)t.get(0)).get("key2").toString().equals("2"));
-
+
Assert.assertTrue(it.hasNext());
t = it.next();
Assert.assertTrue(((Map)t.get(0)).containsKey("key"));
Assert.assertTrue(((Map)t.get(0)).get("key") instanceof Integer);
Assert.assertTrue(((Map)t.get(0)).get("key").toString().equals("2"));
-
+
Assert.assertFalse(it.hasNext());
}
-
+
@Test
public void testComplexLoad() throws IOException, ParseException {
- PigServer pig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer pig = new PigServer(ExecType.LOCAL, cluster.getProperties());
String[] input = {
"[key#{(1,2),(1,3)},134#]",
"[key2#]",
};
-
- Util.createInputFile(cluster, "testComplexLoad", input);
-
- String query = "a = load 'testComplexLoad' as (m:map[bag{(i:int,j:int)}]);";
+
+ Util.createInputFile(FileSystem.getLocal(new Configuration()), tmpDirName + "/testComplexLoad", input);
+
+ String query = "a = load '" + tmpDirName + "/testComplexLoad' as (m:map[bag{(i:int,j:int)}]);";
Util.registerMultiLineQuery(pig, query);
Schema sch = pig.dumpSchema("a");
assertEquals("Checking expected schema",sch.toString(), "{m: map[{(i: int,j: int)}]}");
Iterator<Tuple> it = pig.openIterator("a");
-
+
Assert.assertTrue(it.hasNext());
Tuple t = it.next();
Assert.assertTrue(t.size()==1);
@@ -208,32 +212,32 @@ public class TestTypedMap {
Assert.assertTrue(((Map)t.get(0)).get("key") instanceof DataBag);
Assert.assertTrue(((Map)t.get(0)).get("key").toString().equals("{(1,2),(1,3)}"));
Assert.assertTrue(((Map)t.get(0)).get("134")==null);
-
+
Assert.assertTrue(it.hasNext());
t = it.next();
Assert.assertTrue(((Map)t.get(0)).containsKey("key2"));
Assert.assertTrue(((Map)t.get(0)).get("key2")==null);
-
+
Assert.assertFalse(it.hasNext());
}
-
+
@Test
public void testComplexCast() throws IOException, ParseException {
- PigServer pig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer pig = new PigServer(ExecType.LOCAL, cluster.getProperties());
String[] input = {
"[key#{(1,2),(1,3)},134#]",
"[key2#]",
};
-
- Util.createInputFile(cluster, "testComplexCast", input);
-
- String query = "a = load 'testComplexCast' as (m);" +
+
+ Util.createInputFile(FileSystem.getLocal(new Configuration()), tmpDirName + "/testComplexCast", input);
+
+ String query = "a = load '" + tmpDirName + "/testComplexCast' as (m);" +
"b = foreach a generate ([{(i:int,j:int)}])m;";
Util.registerMultiLineQuery(pig, query);
Schema sch = pig.dumpSchema("b");
assertEquals("Checking expected schema",sch.toString(), "{m: map[{(i: int,j: int)}]}");
Iterator<Tuple> it = pig.openIterator("b");
-
+
Assert.assertTrue(it.hasNext());
Tuple t = it.next();
Assert.assertTrue(t.size()==1);
@@ -243,31 +247,31 @@ public class TestTypedMap {
Assert.assertTrue(((Map)t.get(0)).get("key") instanceof DataBag);
Assert.assertTrue(((Map)t.get(0)).get("key").toString().equals("{(1,2),(1,3)}"));
Assert.assertTrue(((Map)t.get(0)).get("134")==null);
-
+
Assert.assertTrue(it.hasNext());
t = it.next();
Assert.assertTrue(((Map)t.get(0)).containsKey("key2"));
Assert.assertTrue(((Map)t.get(0)).get("key2")==null);
-
+
Assert.assertFalse(it.hasNext());
}
-
+
@Test
public void testComplexCast2() throws IOException, ParseException {
- PigServer pig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer pig = new PigServer(ExecType.LOCAL, cluster.getProperties());
String[] input = {
"[key#1,key2#2]",
};
-
- Util.createInputFile(cluster, "testComplexCast2", input);
-
- String query = "a = load 'testComplexCast2' as (m:[int]);" +
+
+ Util.createInputFile(FileSystem.getLocal(new Configuration()), tmpDirName + "/testComplexCast2", input);
+
+ String query = "a = load '" + tmpDirName + "/testComplexCast2' as (m:[int]);" +
"b = foreach a generate ([long])m;";
Util.registerMultiLineQuery(pig, query);
Schema sch = pig.dumpSchema("b");
assertEquals("Checking expected schema",sch.toString(), "{m: map[long]}");
Iterator<Tuple> it = pig.openIterator("b");
-
+
Assert.assertTrue(it.hasNext());
Tuple t = it.next();
Assert.assertTrue(t.size()==1);
@@ -278,25 +282,25 @@ public class TestTypedMap {
Assert.assertTrue(((Map)t.get(0)).get("key").toString().equals("1"));
Assert.assertTrue(((Map)t.get(0)).get("key2") instanceof Long);
Assert.assertTrue(((Map)t.get(0)).get("key2").toString().equals("2"));
-
+
Assert.assertFalse(it.hasNext());
}
-
+
@Test
public void testUnTypedMap() throws IOException, ParseException {
- PigServer pig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer pig = new PigServer(ExecType.LOCAL, cluster.getProperties());
String[] input = {
"[key#1,key2#2]",
};
-
- Util.createInputFile(cluster, "testUnTypedMap", input);
-
- String query = "a = load 'testUnTypedMap' as (m:[]);";
+
+ Util.createInputFile(FileSystem.getLocal(new Configuration()), tmpDirName + "/testUnTypedMap", input);
+
+ String query = "a = load '" + tmpDirName + "/testUnTypedMap' as (m:[]);";
Util.registerMultiLineQuery(pig, query);
Schema sch = pig.dumpSchema("a");
assertEquals("Checking expected schema",sch.toString(), "{m: map[]}");
Iterator<Tuple> it = pig.openIterator("a");
-
+
Assert.assertTrue(it.hasNext());
Tuple t = it.next();
Assert.assertTrue(t.size()==1);
@@ -307,56 +311,56 @@ public class TestTypedMap {
Assert.assertTrue(((Map)t.get(0)).get("key").toString().equals("1"));
Assert.assertTrue(((Map)t.get(0)).get("key2") instanceof DataByteArray);
Assert.assertTrue(((Map)t.get(0)).get("key2").toString().equals("2"));
-
+
Assert.assertFalse(it.hasNext());
}
-
+
@Test
public void testOrderBy() throws IOException, ParseException {
- PigServer pig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ PigServer pig = new PigServer(ExecType.LOCAL, cluster.getProperties());
String[] input = {
"[key#1,key1#2]",
"[key#2,key3#2]",
"[key#11]",
};
-
- Util.createInputFile(cluster, "testOrderBy", input);
-
- String query = "a = load 'testOrderBy' as (m:[int]);" +
+
+ Util.createInputFile(FileSystem.getLocal(new Configuration()), tmpDirName + "/testOrderBy", input);
+
+ String query = "a = load '" + tmpDirName + "/testOrderBy' as (m:[int]);" +
"b = foreach a generate m#'key' as b0;" +
"c = order b by b0;";
Util.registerMultiLineQuery(pig, query);
Iterator<Tuple> it = pig.openIterator("c");
-
+
Assert.assertTrue(it.hasNext());
Tuple t = it.next();
Assert.assertTrue(t.toString().equals("(1)"));
-
+
Assert.assertTrue(it.hasNext());
t = it.next();
Assert.assertTrue(t.toString().equals("(2)"));
-
+
Assert.assertTrue(it.hasNext());
t = it.next();
Assert.assertTrue(t.toString().equals("(11)"));
-
+
Assert.assertFalse(it.hasNext());
-
+
// Untyped map will sort by byte type, which is different
- query = "a = load 'testOrderBy' as (m:[]);" +
+ query = "a = load '" + tmpDirName + "/testOrderBy' as (m:[]);" +
"b = foreach a generate m#'key' as b0;" +
"c = order b by b0;";
Util.registerMultiLineQuery(pig, query);
it = pig.openIterator("c");
-
+
Assert.assertTrue(it.hasNext());
t = it.next();
Assert.assertTrue(t.toString().equals("(1)"));
-
+
Assert.assertTrue(it.hasNext());
t = it.next();
Assert.assertTrue(t.toString().equals("(11)"));
-
+
Assert.assertTrue(it.hasNext());
t = it.next();
Assert.assertTrue(t.toString().equals("(2)"));