You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by kn...@apache.org on 2016/05/12 16:42:25 UTC
svn commit: r1743526 - in /pig/trunk: CHANGES.txt
test/org/apache/pig/test/TestBuiltin.java
Author: knoguchi
Date: Thu May 12 16:42:25 2016
New Revision: 1743526
URL: http://svn.apache.org/viewvc?rev=1743526&view=rev
Log:
PIG-4881: TestBuiltin.testUniqueID failing on hadoop-1.x (knoguchi)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/test/org/apache/pig/test/TestBuiltin.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1743526&r1=1743525&r2=1743526&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu May 12 16:42:25 2016
@@ -119,6 +119,8 @@ PIG-4639: Add better parser for Apache H
BUG FIXES
+PIG-4881: TestBuiltin.testUniqueID failing on hadoop-1.x (knoguchi)
+
PIG-4888: Line number off when reporting syntax error inside a macro (knoguchi)
PIG-3772: Syntax error when casting an inner schema of a bag and line break involved (ssvinarchukhorton via knoguchi)
Modified: pig/trunk/test/org/apache/pig/test/TestBuiltin.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestBuiltin.java?rev=1743526&r1=1743525&r2=1743526&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestBuiltin.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestBuiltin.java Thu May 12 16:42:25 2016
@@ -130,6 +130,7 @@ import org.apache.pig.data.DefaultBagFac
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
import org.apache.pig.impl.PigContext;
+import org.apache.pig.impl.io.FileLocalizer;
import org.apache.pig.impl.io.ReadToEndLoader;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
@@ -3206,44 +3207,47 @@ public class TestBuiltin {
@Test
public void testUniqueID() throws Exception {
Util.resetStateForExecModeSwitch();
- String inputFileName = "testUniqueID.txt";
- Util.createInputFile(cluster, inputFileName, new String[]
- {"1\n2\n3\n4\n5\n1\n2\n3\n4\n5\n"});
Properties copyproperties = new Properties();
copyproperties.putAll(cluster.getProperties());
PigServer pigServer = new PigServer(cluster.getExecType(), copyproperties);
- pigServer.getPigContext().getProperties().setProperty("mapred.max.split.size", "10");
+
+ // running with 2 mappers each taking 5 records
+ String TMP_DIR = FileLocalizer.getTemporaryPath(pigServer.getPigContext()).toUri().getPath();
+ Util.createInputFile(cluster, TMP_DIR + "/input1.txt", new String[] {"1\n2\n3\n4\n5"});
+ Util.createInputFile(cluster, TMP_DIR + "/input2.txt", new String[] {"1\n2\n3\n4\n5"});
pigServer.getPigContext().getProperties().setProperty("pig.noSplitCombination", "true");
- pigServer.registerQuery("A = load '" + inputFileName + "' as (name);");
+
+ pigServer.registerQuery("A = load '" + TMP_DIR + "' as (name);");
pigServer.registerQuery("B = foreach A generate name, UniqueID();");
Iterator<Tuple> iter = pigServer.openIterator("B");
- assertEquals(iter.next().get(1),"0-0");
- assertEquals(iter.next().get(1),"0-1");
- assertEquals(iter.next().get(1),"0-2");
- assertEquals(iter.next().get(1),"0-3");
- assertEquals(iter.next().get(1),"0-4");
- assertEquals(iter.next().get(1),"1-0");
- assertEquals(iter.next().get(1),"1-1");
- assertEquals(iter.next().get(1),"1-2");
- assertEquals(iter.next().get(1),"1-3");
- assertEquals(iter.next().get(1),"1-4");
- Util.deleteFile(cluster, inputFileName);
+ assertEquals("0-0",iter.next().get(1));
+ assertEquals("0-1",iter.next().get(1));
+ assertEquals("0-2",iter.next().get(1));
+ assertEquals("0-3",iter.next().get(1));
+ assertEquals("0-4",iter.next().get(1));
+ assertEquals("1-0",iter.next().get(1));
+ assertEquals("1-1",iter.next().get(1));
+ assertEquals("1-2",iter.next().get(1));
+ assertEquals("1-3",iter.next().get(1));
+ assertEquals("1-4",iter.next().get(1));
+ Util.deleteFile(cluster, TMP_DIR + "/input1.txt");
+ Util.deleteFile(cluster, TMP_DIR + "/input2.txt");
}
@Test
public void testRANDOMWithJob() throws Exception {
Util.resetStateForExecModeSwitch();
- String inputFileName = "testRANDOM.txt";
- Util.createInputFile(cluster, inputFileName, new String[]
- {"1\n2\n3\n4\n5\n1\n2\n3\n4\n5\n"});
-
Properties copyproperties = new Properties();
copyproperties.putAll(cluster.getProperties());
PigServer pigServer = new PigServer(cluster.getExecType(), copyproperties);
- // running with two mappers
- pigServer.getPigContext().getProperties().setProperty("mapred.max.split.size", "10");
+
+ // running with 2 mappers each taking 5 records
+ String TMP_DIR = FileLocalizer.getTemporaryPath(pigServer.getPigContext()).toUri().getPath();
+ Util.createInputFile(cluster, TMP_DIR + "/input1.txt", new String[] {"1\n2\n3\n4\n5"});
+ Util.createInputFile(cluster, TMP_DIR + "/input2.txt", new String[] {"1\n2\n3\n4\n5"});
pigServer.getPigContext().getProperties().setProperty("pig.noSplitCombination", "true");
- pigServer.registerQuery("A = load '" + inputFileName + "' as (name);");
+
+ pigServer.registerQuery("A = load '" + TMP_DIR + "' as (name);");
pigServer.registerQuery("B = foreach A generate name, RANDOM();");
Iterator<Tuple> iter = pigServer.openIterator("B");
double [] mapper1 = new double[5];
@@ -3266,7 +3270,8 @@ public class TestBuiltin {
for( int i = 0; i < 5; i++ ){
assertNotEquals(mapper1[i], mapper2[i], 0.0001);
}
- Util.deleteFile(cluster, inputFileName);
+ Util.deleteFile(cluster, TMP_DIR + "/input1.txt");
+ Util.deleteFile(cluster, TMP_DIR + "/input2.txt");
}