You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2015/09/17 04:41:04 UTC
svn commit: r1703483 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/builtin/TOMAP.java
test/org/apache/pig/builtin/TestTOMAP.java
test/org/apache/pig/test/TestBuiltin.java
Author: daijy
Date: Thu Sep 17 02:41:02 2015
New Revision: 1703483
URL: http://svn.apache.org/r1703483
Log:
PIG-4674: TOMAP should infer schema
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/builtin/TOMAP.java
pig/trunk/test/org/apache/pig/builtin/TestTOMAP.java
pig/trunk/test/org/apache/pig/test/TestBuiltin.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1703483&r1=1703482&r2=1703483&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Sep 17 02:41:02 2015
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
IMPROVEMENTS
+PIG-4674: TOMAP should infer schema (daijy)
+
PIG-4676: Upgrade Hive to 1.2.1 (daijy)
PIG-4574: Eliminate identity vertex for order by and skewed join right after LOAD (rohini)
Modified: pig/trunk/src/org/apache/pig/builtin/TOMAP.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/builtin/TOMAP.java?rev=1703483&r1=1703482&r2=1703483&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/builtin/TOMAP.java (original)
+++ pig/trunk/src/org/apache/pig/builtin/TOMAP.java Thu Sep 17 02:41:02 2015
@@ -18,7 +18,6 @@
package org.apache.pig.builtin;
import java.io.IOException;
-import java.util.Iterator;
import java.util.Map;
import java.util.HashMap;
@@ -86,7 +85,34 @@ public class TOMAP extends EvalFunc<Map>
@Override
public Schema outputSchema(Schema input) {
- return new Schema(new Schema.FieldSchema(null, DataType.MAP));
+ Byte valueType = null;
+ if (input.size() == 1) {
+ // If input is bag with 'pair' tuples
+ Schema bagSchema = input.getFields().get(0).schema;
+ if (bagSchema != null && bagSchema.size() == 1) {
+ Schema tupleSchema = bagSchema.getFields().get(0).schema;
+ if (tupleSchema != null) {
+ valueType = tupleSchema.getFields().get(1).type;
+ }
+ }
+ } else if (input != null && input.getFields()!=null) {
+ for (int i=0;i<input.size();i++) {
+ if (i % 2 == 1) {
+ if (valueType == null) {
+ valueType = input.getFields().get(i).type;
+ } else if (valueType != input.getFields().get(i).type) {
+ valueType = DataType.BYTEARRAY;
+ break;
+ }
+ }
+ }
+ }
+ Schema s = new Schema(new Schema.FieldSchema(null, DataType.MAP));
+ if (valueType != DataType.BYTEARRAY) {
+ s.getFields().get(0).schema = new Schema(new Schema.FieldSchema(null, valueType));
+ return s;
+ }
+ return s;
}
@Override
Modified: pig/trunk/test/org/apache/pig/builtin/TestTOMAP.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/builtin/TestTOMAP.java?rev=1703483&r1=1703482&r2=1703483&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/builtin/TestTOMAP.java (original)
+++ pig/trunk/test/org/apache/pig/builtin/TestTOMAP.java Thu Sep 17 02:41:02 2015
@@ -49,9 +49,10 @@ public class TestTOMAP {
tuple("a", "b", "c", "d")
);
- pigServer.registerQuery("A = LOAD 'foo' USING mock.Storage();");
+ pigServer.registerQuery("A = LOAD 'foo' USING mock.Storage() as (a0:chararray, a1:chararray, a2:chararray, a3:chararray);");
pigServer.registerQuery("B = FOREACH A GENERATE TOMAP($0, $1, $2, $3);");
pigServer.registerQuery("STORE B INTO 'bar' USING mock.Storage();");
+ assertEquals(pigServer.dumpSchema("B").toString(), "{map[chararray]}");
List<Tuple> out = data.get("bar");
assertEquals(tuple(map("c", "d", "a", "b")), out.get(0));
@@ -82,10 +83,11 @@ public class TestTOMAP {
tuple("c", "d")
);
- pigServer.registerQuery("A = LOAD 'foo' USING mock.Storage();");
+ pigServer.registerQuery("A = LOAD 'foo' USING mock.Storage() as (a0:chararray, a1:chararray);");
pigServer.registerQuery("B = GROUP A ALL;");
pigServer.registerQuery("C = FOREACH B GENERATE TOMAP(A);");
pigServer.registerQuery("STORE C INTO 'bar' USING mock.Storage();");
+ assertEquals(pigServer.dumpSchema("C").toString(), "{map[chararray]}");
List<Tuple> out = data.get("bar");
assertEquals(tuple(map("a", "b", "c", "d")), out.get(0));
Modified: pig/trunk/test/org/apache/pig/test/TestBuiltin.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestBuiltin.java?rev=1703483&r1=1703482&r2=1703483&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestBuiltin.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestBuiltin.java Thu Sep 17 02:41:02 2015
@@ -3201,4 +3201,28 @@ public class TestBuiltin {
iter.next().get(1).equals("1-3");
iter.next().get(1).equals("1-4");
}
+
+ @Test
+ public void testToMapSchema() throws Exception {
+ PigServer pigServer = new PigServer(Util.getLocalTestMode(), new Properties());
+ pigServer.registerQuery("A = load '1.txt' as (a0:chararray, a1:int, a2:double, a3);");
+ pigServer.registerQuery("B = foreach A generate [a0,a1];");
+ Schema s = pigServer.dumpSchema("B");
+ Assert.assertEquals(s.toString(), "{map[int]}");
+ pigServer.registerQuery("B = foreach A generate [a0,a1,a0,a2];");
+ s = pigServer.dumpSchema("B");
+ Assert.assertEquals(s.toString(), "{map[]}");
+ pigServer.registerQuery("B = foreach A generate [a0,a3];");
+ s = pigServer.dumpSchema("B");
+ Assert.assertEquals(s.toString(), "{map[]}");
+ pigServer.registerQuery("A = load '1.txt' as (a:{(a0:chararray, a1:int)};");
+ pigServer.registerQuery("B = foreach A generate TOMAP(a);");
+ s = pigServer.dumpSchema("B");
+ Assert.assertEquals(s.toString(), "{map[int]}");
+ pigServer.registerQuery("A = load '1.txt' as (a:{(a0, a1, a2, a3:int)};");
+ pigServer.registerQuery("B = foreach A generate TOMAP(a);");
+ s = pigServer.dumpSchema("B");
+ Assert.assertEquals(s.toString(), "{map[]}");
+
+ }
}