You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ga...@apache.org on 2011/03/21 21:11:38 UTC
svn commit: r1083930 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/builtin/COUNT.java
test/org/apache/pig/test/TestBuiltin.java
Author: gates
Date: Mon Mar 21 20:11:38 2011
New Revision: 1083930
URL: http://svn.apache.org/viewvc?rev=1083930&view=rev
Log:
PIG-671 typechecker does not throw an error when multiple arguments are passed to COUNT
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/builtin/COUNT.java
pig/trunk/test/org/apache/pig/test/TestBuiltin.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1083930&r1=1083929&r2=1083930&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Mon Mar 21 20:11:38 2011
@@ -118,6 +118,9 @@ PIG-1696: Performance: Use System.arrayc
BUG FIXES
+PIG-671: typechecker does not throw an error when multiple arguments are
+passed to COUNT (deepujain via gates)
+
PIG-1152: bincond operator throws parser error (xuefuz via thejas)
PIG-1885: SUBSTRING fails when input length less than start (deepujain via
Modified: pig/trunk/src/org/apache/pig/builtin/COUNT.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/builtin/COUNT.java?rev=1083930&r1=1083929&r2=1083930&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/builtin/COUNT.java (original)
+++ pig/trunk/src/org/apache/pig/builtin/COUNT.java Mon Mar 21 20:11:38 2011
@@ -18,18 +18,21 @@
package org.apache.pig.builtin;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Iterator;
-import java.util.Map;
+import java.util.List;
import org.apache.pig.Accumulator;
import org.apache.pig.Algebraic;
import org.apache.pig.EvalFunc;
+import org.apache.pig.FuncSpec;
import org.apache.pig.PigException;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.DataType;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
+import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
/**
@@ -149,6 +152,18 @@ public class COUNT extends EvalFunc<Long
return new Schema(new Schema.FieldSchema(null, DataType.LONG));
}
+ /* (non-Javadoc)
+ * @see org.apache.pig.EvalFunc#getArgToFuncMapping()
+ */
+ @Override
+ public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
+ List<FuncSpec> funcList = new ArrayList<FuncSpec>();
+ Schema s = new Schema();
+ s.add(new Schema.FieldSchema(null, DataType.BAG));
+ funcList.add(new FuncSpec(this.getClass().getName(), s));
+ return funcList;
+ }
+
/* Accumulator interface implementation */
private long intermediateCount = 0L;
Modified: pig/trunk/test/org/apache/pig/test/TestBuiltin.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestBuiltin.java?rev=1083930&r1=1083929&r2=1083930&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestBuiltin.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestBuiltin.java Mon Mar 21 20:11:38 2011
@@ -20,6 +20,7 @@ package org.apache.pig.test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import java.io.IOException;
import java.lang.reflect.Method;
import java.util.Arrays;
import java.util.HashMap;
@@ -83,7 +84,9 @@ import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.io.ReadToEndLoader;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
+import org.apache.pig.impl.logicalLayer.validators.TypeCheckerException;
import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -853,6 +856,199 @@ public class TestBuiltin {
}
@Test
+ public void testCount_ValidNumberOfArguments_WithoutInputSchema_One() throws Exception {
+ String inputFileName = createCountInputFile();
+ try {
+ pigServer.registerQuery("A = load '" + inputFileName + "';");
+ pigServer.registerQuery("B = group A all;");
+ pigServer.registerQuery("C = foreach B generate COUNT(A.$0);");
+ assertValidCount();
+ }catch(TypeCheckerException e) {
+ Assert.fail("Query is in accordance with schema, still it failed to execute");
+ }finally {
+ Util.deleteFile(cluster, inputFileName);
+ }
+ }
+
+ @Test
+ public void testCount_ValidNumberOfArguments_WithoutInputSchema_Two() throws Exception {
+ String inputFileName = createCountInputFile();
+ try {
+ pigServer.registerQuery("A = load '" + inputFileName + "';");
+ pigServer.registerQuery("B = group A all;");
+ pigServer.registerQuery("C = foreach B generate COUNT(A);");
+ assertValidCount();
+ }catch(TypeCheckerException e) {
+ Assert.fail("Query is in accordance with schema, still it failed to execute");
+ }finally {
+ Util.deleteFile(cluster, inputFileName);
+ }
+ }
+
+ @Test
+ public void testCount_ValidNumberOfArguments_WithNullTuplesInInput_CaseOne() throws Exception {
+ String inputFileName = "CountTest.txt";
+ String[] inputData = new String[] {
+ "1 2 3 4\n",
+ "\n",
+ "a b c d\n",
+ "\n",
+ "r s t u"};
+ Util.createInputFile(cluster, inputFileName, inputData);
+
+ try {
+ pigServer.registerQuery("A = load '" + inputFileName + "';");
+ pigServer.registerQuery("B = group A all;");
+ pigServer.registerQuery("C = foreach B generate COUNT(A);");
+ assertValidCount();
+ }catch(TypeCheckerException e) {
+ Assert.fail("Query is in accordance with schema, still it failed to execute");
+ }finally {
+ Util.deleteFile(cluster, inputFileName);
+ }
+ }
+
+ @Test
+ public void testCount_ValidNumberOfArguments_WithNullTuplesInInput_CaseTwo() throws Exception {
+ String inputFileName = "CountTest.txt";
+ String[] inputData = new String[] {
+ "1 2 3 4\n",
+ "\n",
+ "a b c d\n",
+ "\n",
+ "r s t u"};
+ Util.createInputFile(cluster, inputFileName, inputData);
+
+ try {
+ pigServer.registerQuery("A = load '" + inputFileName + "';");
+ pigServer.registerQuery("B = group A all;");
+ pigServer.registerQuery("C = foreach B generate COUNT(A.$0);");
+ assertValidCount();
+ }catch(TypeCheckerException e) {
+ Assert.fail("Query is in accordance with schema, still it failed to execute");
+ }finally {
+ Util.deleteFile(cluster, inputFileName);
+ }
+ }
+
+
+ @Test
+ public void testCount_ValidNumberOfArguments_WithInputSchema_One() throws Exception {
+ String inputFileName = createCountInputFile();
+ try {
+ pigServer.registerQuery("A = load '" + inputFileName + "' as (data:chararray);");
+ pigServer.registerQuery("B = group A all;");
+ pigServer.registerQuery("C = foreach B generate COUNT(A.$0);");
+ assertValidCount();
+ }catch(TypeCheckerException e) {
+ Assert.fail("Query is in accordance with schema, still it failed to execute");
+ }finally {
+ Util.deleteFile(cluster, inputFileName);
+ }
+ }
+
+ @Test
+ public void testCount_ValidNumberOfArguments_WithInputSchema_Two() throws Exception {
+ String inputFileName = createCountInputFile();
+ try {
+ pigServer.registerQuery("A = load '" + inputFileName + "' as (data:chararray);");
+ pigServer.registerQuery("B = group A all;");
+ pigServer.registerQuery("C = foreach B generate COUNT(A);");
+ assertValidCount();
+ }catch(TypeCheckerException e) {
+ Assert.fail("Query is in accordance with schema, still it failed to execute");
+ }finally {
+ Util.deleteFile(cluster, inputFileName);
+ }
+ }
+
+ /**
+ * Assert the result of COUNT for valid scenarios has only one tuple in output which has only one item which is set
+ * to 3.
+ *
+ * @throws IOException
+ * @throws ExecException
+ */
+ private void assertValidCount() throws IOException, ExecException {
+ Iterator<Tuple> it = pigServer.openIterator("C");
+ int i=0;
+ final int expectedOutputTupleSize = 1;
+ final long expectedCount = 3;
+ while (it.hasNext()) {
+ Tuple t = it.next();
+ assertEquals("Testing SIZE(<Tuple>): ", expectedOutputTupleSize, t.size());
+ assertEquals("Testing Value within<Tuple>: ", expectedCount, t.get(0));
+ i++;
+ }
+ assertEquals("Testing the above loop ran only once.",1,i);
+ }
+
+ @Test
+ public void testCount_InvalidNumberOfArguments_WithoutInputSchema() throws Exception {
+ String inputFileName = createCountInputFile();
+ try {
+ pigServer.registerQuery("A = load '" + inputFileName + "';");
+ pigServer.registerQuery("B = group A all;");
+ pigServer.registerQuery("C = foreach B generate COUNT(A.$0, A.$0);");
+ pigServer.openIterator("C");
+ Assert.fail("COUNT is suppose to run with one argument of type BAG, however it ran with couple of arguments.");
+ }catch(TypeCheckerException e) {
+
+ }finally {
+ Util.deleteFile(cluster, inputFileName);
+ }
+ }
+
+ @Test
+ public void testCount_InvalidNumberOfArguments_WithInputSchema() throws Exception {
+ String inputFileName = createCountInputFile();
+ try {
+ pigServer.registerQuery("A = load '" + inputFileName + "' as (data:chararray);");
+ pigServer.registerQuery("B = group A all;");
+ pigServer.registerQuery("C = foreach B generate COUNT(A.$0, A.$0);");
+ pigServer.openIterator("C");
+ Assert.fail("COUNT is suppose to run with one argument of type BAG, however it ran with couple of arguments.");
+ }catch(TypeCheckerException e) {
+
+ }finally {
+ Util.deleteFile(cluster, inputFileName);
+ }
+ }
+
+ @Test
+ public void testCount_InvalidArgumentType() throws Exception {
+ String inputFileName = createCountInputFile();
+ try {
+ pigServer.registerQuery("A = load '" + inputFileName + "' as (data:chararray);");
+ pigServer.registerQuery("B = group A all;");
+ pigServer.registerQuery("C = foreach B generate COUNT('data');");
+ pigServer.openIterator("C");
+ Assert.fail("COUNT is suppose to run with one argument of type BAG, however it ran with an argument of type chararray.");
+ }catch(TypeCheckerException e) {
+
+ }finally {
+ Util.deleteFile(cluster, inputFileName);
+ }
+ }
+
+ /**
+ * Creates a input file that will be used to test COUNT builtin function.
+ *
+ * @return name of the input file.
+ * @throws IOException
+ * Unable to create input file.
+ */
+ protected String createCountInputFile() throws IOException {
+ String inputFileName = "CountTest.txt";
+ String[] inputData = new String[] {
+ "1 2 3 4\n",
+ "a b c d\n",
+ "r s t u"};
+ Util.createInputFile(cluster, inputFileName, inputData);
+ return inputFileName;
+ }
+
+ @Test
public void testCOUNT_STAR() throws Exception {
Integer input[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, null };
long expected = input.length;