You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by sm...@apache.org on 2009/07/09 03:14:11 UTC
svn commit: r792368 - in /hadoop/pig/trunk: CHANGES.txt
src/org/apache/pig/data/DataType.java
test/org/apache/pig/test/TestUDFReturnMap.java
test/org/apache/pig/test/utils/MyUDFReturnMap.java
Author: sms
Date: Thu Jul 9 01:14:10 2009
New Revision: 792368
URL: http://svn.apache.org/viewvc?rev=792368&view=rev
Log:
PIG-851: PIG-851: Map type used as return type in UDFs not recognized at all times (zjffdu via sms)
Added:
hadoop/pig/trunk/test/org/apache/pig/test/TestUDFReturnMap.java
hadoop/pig/trunk/test/org/apache/pig/test/utils/MyUDFReturnMap.java
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/data/DataType.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=792368&r1=792367&r2=792368&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Thu Jul 9 01:14:10 2009
@@ -40,6 +40,9 @@
BUG FIXES
+ PIG-851: Map type used as return type in UDFs not recognized at all times
+ (zjffdu via sms)
+
PIG-861: POJoinPackage lose tuple in large dataset (daijy)
PIG-797: Limit with ORDER BY producing wrong results (daijy)
Modified: hadoop/pig/trunk/src/org/apache/pig/data/DataType.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/data/DataType.java?rev=792368&r1=792367&r2=792368&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/data/DataType.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/data/DataType.java Thu Jul 9 01:14:10 2009
@@ -17,27 +17,22 @@
*/
package org.apache.pig.data;
-import java.lang.Class;
+import java.lang.reflect.ParameterizedType;
import java.lang.reflect.Type;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.TreeMap;
-import java.util.ArrayList;
-import org.apache.hadoop.io.BooleanWritable;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.WritableComparable;
import org.apache.pig.PigException;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.logicalLayer.schema.SchemaMergeException;
+
+
/**
* A class of static final values used to encode data type and a number of
* static helper funcitons for manipulating data objects. The data type
@@ -116,31 +111,41 @@
// Might be a tuple or a bag, need to check the interfaces it
// implements
if (t instanceof Class) {
- Class c = (Class)t;
- Class[] ioeInterfaces = c.getInterfaces();
- Class[] interfaces = null;
- if(c.isInterface()){
- interfaces = new Class[ioeInterfaces.length+1];
- interfaces[0] = c;
- for (int i = 1; i < interfaces.length; i++) {
- interfaces[i] = ioeInterfaces[i-1];
- }
- } else {
- interfaces = ioeInterfaces;
- }
- for (int i = 0; i < interfaces.length; i++) {
- if (interfaces[i].getName().equals("org.apache.pig.data.Tuple")) {
- return TUPLE;
- } else if (interfaces[i].getName().equals("org.apache.pig.data.DataBag")) {
- return BAG;
- } else if (interfaces[i].getName().equals("java.util.Map")) {
- return MAP;
- }
- }
+ return extractTypeFromClass(t);
+ }else if (t instanceof ParameterizedType){
+ ParameterizedType impl=(ParameterizedType)t;
+ Class c=(Class)impl.getRawType();
+ return extractTypeFromClass(c);
}
return ERROR;
}
}
+
+ private static byte extractTypeFromClass(Type t) {
+ Class c = (Class)t;
+ Class[] ioeInterfaces = c.getInterfaces();
+ Class[] interfaces = null;
+ if(c.isInterface()){
+ interfaces = new Class[ioeInterfaces.length+1];
+ interfaces[0] = c;
+ for (int i = 1; i < interfaces.length; i++) {
+ interfaces[i] = ioeInterfaces[i-1];
+ }
+ } else {
+ interfaces = ioeInterfaces;
+ }
+ for (int i = 0; i < interfaces.length; i++) {
+ if (interfaces[i].getName().equals("org.apache.pig.data.Tuple")) {
+ return TUPLE;
+ } else if (interfaces[i].getName().equals("org.apache.pig.data.DataBag")) {
+ return BAG;
+ } else if (interfaces[i].getName().equals("java.util.Map")) {
+ return MAP;
+ }
+ }
+
+ return ERROR;
+ }
public static int numTypes(){
byte[] types = genAllTypes();
Added: hadoop/pig/trunk/test/org/apache/pig/test/TestUDFReturnMap.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestUDFReturnMap.java?rev=792368&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestUDFReturnMap.java (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestUDFReturnMap.java Thu Jul 9 01:14:10 2009
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Map;
+
+import junit.framework.TestCase;
+
+import org.apache.pig.ExecType;
+import org.apache.pig.PigServer;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.io.FileLocalizer;
+import org.apache.pig.test.utils.MyUDFReturnMap;
+import org.junit.Test;
+
+public class TestUDFReturnMap extends TestCase {
+
+ static String[] ScriptStatement = {
+ "A = LOAD 'test/org/apache/pig/test/data/passwd' USING PigStorage();",
+ "B = FOREACH A GENERATE org.apache.pig.test.utils.MyUDFReturnMap(1);" };
+
+ static File TempScriptFile = null;
+
+ static MiniCluster cluster = MiniCluster.buildCluster();
+
+ @Override
+ protected void setUp() throws Exception {
+ TempScriptFile = File.createTempFile("temp_jira_851", ".pig");
+ FileWriter writer = new FileWriter(TempScriptFile);
+ for (String line : ScriptStatement) {
+ writer.write(line + "\n");
+ }
+ writer.close();
+ }
+
+ @Test
+ public void testUDFReturnMap_LocalMode() {
+ try {
+ PigServer pig = new PigServer(ExecType.LOCAL);
+ pig.registerScript(TempScriptFile.getAbsolutePath());
+
+ Iterator<Tuple> iterator = pig.openIterator("B");
+ int index = 0;
+ while (iterator.hasNext()) {
+ Tuple tuple = iterator.next();
+ index++;
+ Map<Object, Object> result = (Map<Object, Object>) tuple.get(0);
+ assertEquals(result, MyUDFReturnMap.map);
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ fail();
+ }
+ }
+
+ @Test
+ public void testUDFReturnMap_MapReduceMode() {
+ try {
+ Util.createInputFile(cluster, "a.txt", new String[] { "dummy",
+ "dummy" });
+ FileLocalizer.deleteTempFiles();
+ PigServer pig = new PigServer(ExecType.MAPREDUCE, cluster
+ .getProperties());
+ pig.registerQuery("A = LOAD 'a.txt';");
+ pig
+ .registerQuery("B = FOREACH A GENERATE org.apache.pig.test.utils.MyUDFReturnMap();");
+
+ Iterator<Tuple> iterator = pig.openIterator("B");
+ int index = 0;
+ while (iterator.hasNext()) {
+ Tuple tuple = iterator.next();
+ index++;
+ Map<Object, Object> result = (Map<Object, Object>) tuple.get(0);
+ assertEquals(result, MyUDFReturnMap.map);
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ fail();
+ }
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ TempScriptFile.delete();
+ }
+}
Added: hadoop/pig/trunk/test/org/apache/pig/test/utils/MyUDFReturnMap.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/utils/MyUDFReturnMap.java?rev=792368&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/utils/MyUDFReturnMap.java (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/utils/MyUDFReturnMap.java Thu Jul 9 01:14:10 2009
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test.utils;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.pig.EvalFunc;
+import org.apache.pig.data.Tuple;
+
+public class MyUDFReturnMap extends EvalFunc<Map<Object,Object>> {
+
+ public static Map<Object,Object> map=new HashMap<Object, Object>();
+
+ static{
+ map.put("anykey", "anyvalue");
+ }
+
+ @Override
+ public Map<Object, Object> exec(Tuple input) throws IOException {
+ return map;
+ }
+
+}