You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ga...@apache.org on 2011/03/21 21:11:38 UTC

svn commit: r1083930 - in /pig/trunk: CHANGES.txt src/org/apache/pig/builtin/COUNT.java test/org/apache/pig/test/TestBuiltin.java

Author: gates
Date: Mon Mar 21 20:11:38 2011
New Revision: 1083930

URL: http://svn.apache.org/viewvc?rev=1083930&view=rev
Log:
PIG-671  typechecker does not throw an error when multiple arguments are passed to COUNT

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/builtin/COUNT.java
    pig/trunk/test/org/apache/pig/test/TestBuiltin.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1083930&r1=1083929&r2=1083930&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Mon Mar 21 20:11:38 2011
@@ -118,6 +118,9 @@ PIG-1696: Performance: Use System.arrayc
 
 BUG FIXES
 
+PIG-671:  typechecker does not throw an error when multiple arguments are
+passed to COUNT (deepujain via gates)
+
 PIG-1152: bincond operator throws parser error (xuefuz via thejas)
 
 PIG-1885: SUBSTRING fails when input length less than start (deepujain via

Modified: pig/trunk/src/org/apache/pig/builtin/COUNT.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/builtin/COUNT.java?rev=1083930&r1=1083929&r2=1083930&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/builtin/COUNT.java (original)
+++ pig/trunk/src/org/apache/pig/builtin/COUNT.java Mon Mar 21 20:11:38 2011
@@ -18,18 +18,21 @@
 package org.apache.pig.builtin;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Iterator;
-import java.util.Map;
+import java.util.List;
 
 import org.apache.pig.Accumulator;
 import org.apache.pig.Algebraic;
 import org.apache.pig.EvalFunc;
+import org.apache.pig.FuncSpec;
 import org.apache.pig.PigException;
 import org.apache.pig.backend.executionengine.ExecException;
 import org.apache.pig.data.DataBag;
 import org.apache.pig.data.DataType;
 import org.apache.pig.data.Tuple;
 import org.apache.pig.data.TupleFactory;
+import org.apache.pig.impl.logicalLayer.FrontendException;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
 
 /**
@@ -149,6 +152,18 @@ public class COUNT extends EvalFunc<Long
         return new Schema(new Schema.FieldSchema(null, DataType.LONG)); 
     }
     
+    /* (non-Javadoc)
+     * @see org.apache.pig.EvalFunc#getArgToFuncMapping()
+     */
+    @Override
+    public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
+        List<FuncSpec> funcList = new ArrayList<FuncSpec>();
+        Schema s = new Schema();
+        s.add(new Schema.FieldSchema(null, DataType.BAG));
+        funcList.add(new FuncSpec(this.getClass().getName(), s));
+        return funcList;
+    }
+    
     /* Accumulator interface implementation */
     private long intermediateCount = 0L;
 

Modified: pig/trunk/test/org/apache/pig/test/TestBuiltin.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestBuiltin.java?rev=1083930&r1=1083929&r2=1083930&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestBuiltin.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestBuiltin.java Mon Mar 21 20:11:38 2011
@@ -20,6 +20,7 @@ package org.apache.pig.test;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
+import java.io.IOException;
 import java.lang.reflect.Method;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -83,7 +84,9 @@ import org.apache.pig.impl.PigContext;
 import org.apache.pig.impl.io.ReadToEndLoader;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
 import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
+import org.apache.pig.impl.logicalLayer.validators.TypeCheckerException;
 import org.junit.AfterClass;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -853,6 +856,199 @@ public class TestBuiltin {
     }
 
     @Test
+    public void testCount_ValidNumberOfArguments_WithoutInputSchema_One() throws Exception {
+         String inputFileName = createCountInputFile();
+         try {
+             pigServer.registerQuery("A = load '" + inputFileName + "';");
+             pigServer.registerQuery("B = group A all;");
+             pigServer.registerQuery("C = foreach B generate COUNT(A.$0);");
+             assertValidCount();
+         }catch(TypeCheckerException e) {
+             Assert.fail("Query is in accordance with schema, still it failed to execute");
+         }finally {
+             Util.deleteFile(cluster, inputFileName);
+         }
+    }
+
+    @Test
+    public void testCount_ValidNumberOfArguments_WithoutInputSchema_Two() throws Exception {
+         String inputFileName = createCountInputFile();
+         try {
+             pigServer.registerQuery("A = load '" + inputFileName + "';");
+             pigServer.registerQuery("B = group A all;");
+             pigServer.registerQuery("C = foreach B generate COUNT(A);");
+             assertValidCount();
+         }catch(TypeCheckerException e) {
+             Assert.fail("Query is in accordance with schema, still it failed to execute");
+         }finally {
+             Util.deleteFile(cluster, inputFileName);
+         }
+    }
+    
+    @Test
+    public void testCount_ValidNumberOfArguments_WithNullTuplesInInput_CaseOne() throws Exception {
+        String inputFileName = "CountTest.txt";
+        String[] inputData = new String[] {
+                "1 2 3 4\n",
+                "\n",
+                "a b c d\n",
+                "\n",
+                "r s t u"};
+        Util.createInputFile(cluster, inputFileName, inputData);
+
+         try {
+             pigServer.registerQuery("A = load '" + inputFileName + "';");
+             pigServer.registerQuery("B = group A all;");
+             pigServer.registerQuery("C = foreach B generate COUNT(A);");
+             assertValidCount();
+         }catch(TypeCheckerException e) {
+             Assert.fail("Query is in accordance with schema, still it failed to execute");
+         }finally {
+             Util.deleteFile(cluster, inputFileName);
+         }
+    }
+    
+    @Test
+    public void testCount_ValidNumberOfArguments_WithNullTuplesInInput_CaseTwo() throws Exception {
+        String inputFileName = "CountTest.txt";
+        String[] inputData = new String[] {
+                "1 2 3 4\n",
+                "\n",
+                "a b c d\n",
+                "\n",
+                "r s t u"};
+        Util.createInputFile(cluster, inputFileName, inputData);
+
+         try {
+             pigServer.registerQuery("A = load '" + inputFileName + "';");
+             pigServer.registerQuery("B = group A all;");
+             pigServer.registerQuery("C = foreach B generate COUNT(A.$0);");
+             assertValidCount();
+         }catch(TypeCheckerException e) {
+             Assert.fail("Query is in accordance with schema, still it failed to execute");
+         }finally {
+             Util.deleteFile(cluster, inputFileName);
+         }
+    }
+    
+    
+    @Test
+    public void testCount_ValidNumberOfArguments_WithInputSchema_One() throws Exception {
+         String inputFileName = createCountInputFile();
+         try {
+             pigServer.registerQuery("A = load '" + inputFileName + "' as (data:chararray);");
+             pigServer.registerQuery("B = group A all;");
+             pigServer.registerQuery("C = foreach B generate COUNT(A.$0);");
+             assertValidCount();
+         }catch(TypeCheckerException e) {
+             Assert.fail("Query is in accordance with schema, still it failed to execute");
+         }finally {
+             Util.deleteFile(cluster, inputFileName);
+         }
+    }
+    
+    @Test
+    public void testCount_ValidNumberOfArguments_WithInputSchema_Two() throws Exception {
+         String inputFileName = createCountInputFile();
+         try {
+             pigServer.registerQuery("A = load '" + inputFileName + "' as (data:chararray);");
+             pigServer.registerQuery("B = group A all;");
+             pigServer.registerQuery("C = foreach B generate COUNT(A);");
+             assertValidCount();
+         }catch(TypeCheckerException e) {
+             Assert.fail("Query is in accordance with schema, still it failed to execute");
+         }finally {
+             Util.deleteFile(cluster, inputFileName);
+         }
+    }
+
+    /**
+     * Assert the result of COUNT for valid scenarios has only one tuple in output which has only one item which is set
+     * to 3.
+     * 
+     * @throws IOException
+     * @throws ExecException
+     */
+    private void assertValidCount() throws IOException, ExecException {
+        Iterator<Tuple> it = pigServer.openIterator("C");
+         int i=0;
+         final int expectedOutputTupleSize = 1;
+         final long expectedCount = 3;
+         while (it.hasNext()) {
+             Tuple t = it.next();
+             assertEquals("Testing SIZE(<Tuple>): ", expectedOutputTupleSize, t.size());
+             assertEquals("Testing Value within<Tuple>: ", expectedCount, t.get(0));
+             i++;
+         }
+         assertEquals("Testing the above loop ran only once.",1,i);
+    }
+   
+    @Test
+    public void testCount_InvalidNumberOfArguments_WithoutInputSchema() throws Exception {
+         String inputFileName = createCountInputFile();
+         try {
+             pigServer.registerQuery("A = load '" + inputFileName + "';");
+             pigServer.registerQuery("B = group A all;");
+             pigServer.registerQuery("C = foreach B generate COUNT(A.$0, A.$0);");
+             pigServer.openIterator("C");
+             Assert.fail("COUNT is suppose to run with one argument of type BAG, however it ran with couple of arguments.");
+         }catch(TypeCheckerException e) {
+
+         }finally {
+             Util.deleteFile(cluster, inputFileName);
+         }
+    }
+
+    @Test
+    public void testCount_InvalidNumberOfArguments_WithInputSchema() throws Exception {
+         String inputFileName = createCountInputFile();
+         try {
+             pigServer.registerQuery("A = load '" + inputFileName + "' as (data:chararray);");
+             pigServer.registerQuery("B = group A all;");
+             pigServer.registerQuery("C = foreach B generate COUNT(A.$0, A.$0);");
+             pigServer.openIterator("C");
+             Assert.fail("COUNT is suppose to run with one argument of type BAG, however it ran with couple of arguments.");
+         }catch(TypeCheckerException e) {
+             
+         }finally {
+             Util.deleteFile(cluster, inputFileName);
+         }
+    }
+    
+    @Test
+    public void testCount_InvalidArgumentType() throws Exception {
+         String inputFileName = createCountInputFile();
+         try {
+             pigServer.registerQuery("A = load '" + inputFileName + "' as (data:chararray);");
+             pigServer.registerQuery("B = group A all;");
+             pigServer.registerQuery("C = foreach B generate COUNT('data');");
+             pigServer.openIterator("C");
+             Assert.fail("COUNT is suppose to run with one argument of type BAG, however it ran with an argument of type chararray.");
+         }catch(TypeCheckerException e) {
+             
+         }finally {
+             Util.deleteFile(cluster, inputFileName);
+         }
+    }
+
+    /**
+     * Creates a input file that will be used to test COUNT builtin function.
+     * 
+     * @return name of the input file.
+     * @throws IOException
+     *             Unable to create input file.
+     */
+    protected String createCountInputFile() throws IOException {
+        String inputFileName = "CountTest.txt";
+         String[] inputData = new String[] {
+                 "1 2 3 4\n",
+                 "a b c d\n",
+                 "r s t u"};
+         Util.createInputFile(cluster, inputFileName, inputData);
+        return inputFileName;
+    }
+
+    @Test
     public void testCOUNT_STAR() throws Exception {
         Integer input[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, null };
         long expected = input.length;