You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2009/06/02 20:51:21 UTC
svn commit: r781121 - in /hadoop/hive/trunk: ./ data/files/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
serde/if/test/
serde/src/gen-java/org/apache/hadoop/hive/serde2/thrift/test/...
Author: namit
Date: Tue Jun 2 18:51:20 2009
New Revision: 781121
URL: http://svn.apache.org/viewvc?rev=781121&view=rev
Log:
HIVE-495. Fix join of a table of ThriftSerDe with complex columns
(Zheng Shao via namit)
Added:
hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_thrift.q
hadoop/hive/trunk/ql/src/test/results/clientpositive/join_thrift.q.out
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/data/files/complex.seq
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java
hadoop/hive/trunk/ql/src/test/results/clientpositive/case_sensitivity.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/describe_xpath.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input17.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input5.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath2.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath3.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath4.q.out
hadoop/hive/trunk/serde/if/test/complex.thrift
hadoop/hive/trunk/serde/src/gen-java/org/apache/hadoop/hive/serde2/thrift/test/IntString.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java
hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorUtils.java
hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/thrift_test/CreateSequenceFile.java
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=781121&r1=781120&r2=781121&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Tue Jun 2 18:51:20 2009
@@ -162,7 +162,7 @@
the expression string is empty. (Zheng Shao via athusoo)
HIVE-410: Heartbeating for streaming jobs should not depend on stdout.
- (Ashish Thusoo via njain)
+ (Ashish Thusoo via namit)
HIVE-499. Fix for CAST operator on the same type. (Zheng Shao via athusoo)
@@ -181,7 +181,7 @@
HIVE-500. Fix select from newly created table.
(Yongqiang He via zshao)
- HIVE-451. Fix ORDER BY xxx DESC (He Yongqiang via njain)
+ HIVE-451. Fix ORDER BY xxx DESC (He Yongqiang via namit)
HIVE-467. Scratch data location should be on different filesystems for
different types of intermediate data. (Joydeep Sen Sarma via rmurthy)
@@ -200,6 +200,9 @@
HIVE-525. Fix Eclipse classpath by adding Hadoop test jar.
(Prasad Chakka via johan)
+ HIVE-495. Fix join of a table of ThriftSerDe with complex columns
+ (Zheng Shao via namit)
+
Release 0.3.1 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/hive/trunk/data/files/complex.seq
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/data/files/complex.seq?rev=781121&r1=781120&r2=781121&view=diff
==============================================================================
Binary files - no diff available.
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java?rev=781121&r1=781120&r2=781121&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java Tue Jun 2 18:51:20 2009
@@ -37,6 +37,7 @@
import org.apache.hadoop.hive.ql.plan.joinDesc;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject;
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
@@ -44,6 +45,7 @@
import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.io.Text;
@@ -161,8 +163,12 @@
for (Byte alias : order) {
int sz = map.get(alias).size();
StructObjectInspector fldObjIns = (StructObjectInspector)((StructObjectInspector)inputObjInspector[alias.intValue()]).getStructFieldRef("VALUE").getFieldObjectInspector();
- for (int i = 0; i < sz; i++)
- structFieldObjectInspectors.add(fldObjIns.getAllStructFieldRefs().get(i).getFieldObjectInspector());
+ for (int i = 0; i < sz; i++) {
+ structFieldObjectInspectors.add(
+ ObjectInspectorUtils.getStandardObjectInspector(
+ fldObjIns.getAllStructFieldRefs().get(i).getFieldObjectInspector(),
+ ObjectInspectorCopyOption.KEEP));
+ }
}
joinOutputObjectInspector = ObjectInspectorFactory
Added: hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_thrift.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_thrift.q?rev=781121&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_thrift.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_thrift.q Tue Jun 2 18:51:20 2009
@@ -0,0 +1,12 @@
+DESCRIBE src_thrift;
+
+EXPLAIN
+SELECT s1.aint, s2.lintstring
+FROM src_thrift s1
+JOIN src_thrift s2
+ON s1.aint = s2.aint;
+
+SELECT s1.aint, s2.lintstring
+FROM src_thrift s1
+JOIN src_thrift s2
+ON s1.aint = s2.aint;
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/case_sensitivity.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/case_sensitivity.q.out?rev=781121&r1=781120&r2=781121&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/case_sensitivity.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/case_sensitivity.q.out Tue Jun 2 18:51:20 2009
@@ -19,7 +19,7 @@
expr: lint
type: array<int>
expr: lintstring
- type: array<struct<myint:int,mystring:string>>
+ type: array<struct<myint:int,mystring:string,underscore_int:int>>
Filter Operator
predicate:
expr: (0[0] > 0)
@@ -56,7 +56,7 @@
Output: default/dest1
query: SELECT DEST1.* FROM Dest1
Input: default/dest1
-Output: /data/users/athusoo/commits/hive_trunk_ws8/ql/../build/ql/tmp/256252840/400813971.10000
+Output: /data/users/zshao/tools/495-trunk-apache-hive/ql/../build/ql/tmp/154752222/89798452.10000
2 1
4 8
6 27
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/describe_xpath.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/describe_xpath.q.out?rev=781121&r1=781120&r2=781121&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/describe_xpath.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/describe_xpath.q.out Tue Jun 2 18:51:20 2009
@@ -14,6 +14,7 @@
describe src_thrift.lintString.$elem$
myint int from deserializer
mystring string from deserializer
+underscore_int int from deserializer
query: -- Describe a member of an element of a list
describe src_thrift.lintString.$elem$.myint
myint int from deserializer
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/input17.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/input17.q.out?rev=781121&r1=781120&r2=781121&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/input17.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/input17.q.out Tue Jun 2 18:51:20 2009
@@ -26,13 +26,13 @@
expr: lint
type: array<int>
expr: lintstring
- type: array<struct<myint:int,mystring:string>>
+ type: array<struct<myint:int,mystring:string,underscore_int:int>>
Select Operator
expressions:
expr: (0 + 1[0])
type: int
expr: 2[0]
- type: struct<myint:int,mystring:string>
+ type: struct<myint:int,mystring:string,underscore_int:int>
Transform Operator
command: /bin/cat
output info:
@@ -97,15 +97,15 @@
Output: default/dest1
query: SELECT dest1.* FROM dest1
Input: default/dest1
-Output: /data/users/athusoo/commits/hive_trunk_ws8/ql/../build/ql/tmp/10910343/1742990479.10000
-NULL {"myint":null,"mystring":null}
--1461153966 {"myint":49,"mystring":"343"}
--1952710705 {"myint":25,"mystring":"125"}
--734328905 {"myint":16,"mystring":"64"}
--751827636 {"myint":4,"mystring":"8"}
-1244525196 {"myint":36,"mystring":"216"}
-1638581586 {"myint":64,"mystring":"512"}
-1712634731 {"myint":0,"mystring":"0"}
-336964422 {"myint":81,"mystring":"729"}
-465985201 {"myint":1,"mystring":"1"}
-477111225 {"myint":9,"mystring":"27"}
+Output: /data/users/zshao/tools/495-trunk-apache-hive/ql/../build/ql/tmp/899544746/1236316343.10000
+NULL {"myint":null,"mystring":null,"underscore_int":null}
+-1461153966 {"myint":49,"mystring":"343","underscore_int":7}
+-1952710705 {"myint":25,"mystring":"125","underscore_int":5}
+-734328905 {"myint":16,"mystring":"64","underscore_int":4}
+-751827636 {"myint":4,"mystring":"8","underscore_int":2}
+1244525196 {"myint":36,"mystring":"216","underscore_int":6}
+1638581586 {"myint":64,"mystring":"512","underscore_int":8}
+1712634731 {"myint":0,"mystring":"0","underscore_int":0}
+336964422 {"myint":81,"mystring":"729","underscore_int":9}
+465985201 {"myint":1,"mystring":"1","underscore_int":1}
+477111225 {"myint":9,"mystring":"27","underscore_int":3}
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/input5.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/input5.q.out?rev=781121&r1=781120&r2=781121&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/input5.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/input5.q.out Tue Jun 2 18:51:20 2009
@@ -24,13 +24,13 @@
expr: lint
type: array<int>
expr: lintstring
- type: array<struct<myint:int,mystring:string>>
+ type: array<struct<myint:int,mystring:string,underscore_int:int>>
Select Operator
expressions:
expr: 0
type: array<int>
expr: 1
- type: array<struct<myint:int,mystring:string>>
+ type: array<struct<myint:int,mystring:string,underscore_int:int>>
Transform Operator
command: /bin/cat
output info:
@@ -89,15 +89,15 @@
Output: default/dest1
query: SELECT dest1.* FROM dest1
Input: default/dest1
-Output: /data/users/athusoo/commits/hive_trunk_ws8/ql/../build/ql/tmp/248520134/578564891.10000
-[0,0,0] [{"myint":0,"mystring":"0"}]
-[1,2,3] [{"myint":1,"mystring":"1"}]
-[2,4,6] [{"myint":4,"mystring":"8"}]
-[3,6,9] [{"myint":9,"mystring":"27"}]
-[4,8,12] [{"myint":16,"mystring":"64"}]
-[5,10,15] [{"myint":25,"mystring":"125"}]
-[6,12,18] [{"myint":36,"mystring":"216"}]
-[7,14,21] [{"myint":49,"mystring":"343"}]
-[8,16,24] [{"myint":64,"mystring":"512"}]
-[9,18,27] [{"myint":81,"mystring":"729"}]
+Output: /data/users/zshao/tools/495-trunk-apache-hive/ql/../build/ql/tmp/1082689233/217248273.10000
+[0,0,0] [{"myint":0,"mystring":"0","underscore_int":0}]
+[1,2,3] [{"myint":1,"mystring":"1","underscore_int":1}]
+[2,4,6] [{"myint":4,"mystring":"8","underscore_int":2}]
+[3,6,9] [{"myint":9,"mystring":"27","underscore_int":3}]
+[4,8,12] [{"myint":16,"mystring":"64","underscore_int":4}]
+[5,10,15] [{"myint":25,"mystring":"125","underscore_int":5}]
+[6,12,18] [{"myint":36,"mystring":"216","underscore_int":6}]
+[7,14,21] [{"myint":49,"mystring":"343","underscore_int":7}]
+[8,16,24] [{"myint":64,"mystring":"512","underscore_int":8}]
+[9,18,27] [{"myint":81,"mystring":"729","underscore_int":9}]
null null
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath.q.out?rev=781121&r1=781120&r2=781121&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath.q.out Tue Jun 2 18:51:20 2009
@@ -19,7 +19,7 @@
expr: lint
type: array<int>
expr: lintstring
- type: array<struct<myint:int,mystring:string>>
+ type: array<struct<myint:int,mystring:string,underscore_int:int>>
expr: mstringstring
type: map<string,string>
Select Operator
@@ -56,7 +56,7 @@
Output: default/dest1
query: SELECT dest1.* FROM dest1
Input: default/dest1
-Output: /data/users/athusoo/commits/hive_trunk_ws8/ql/../build/ql/tmp/403047878/395453459.10000
+Output: /data/users/zshao/tools/495-trunk-apache-hive/ql/../build/ql/tmp/600192558/110692382.10000
0 0 NULL
2 1 NULL
4 8 value_2
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath2.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath2.q.out?rev=781121&r1=781120&r2=781121&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath2.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath2.q.out Tue Jun 2 18:51:20 2009
@@ -19,7 +19,7 @@
expr: lint
type: array<int>
expr: lintstring
- type: array<struct<myint:int,mystring:string>>
+ type: array<struct<myint:int,mystring:string,underscore_int:int>>
expr: mstringstring
type: map<string,string>
Filter Operator
@@ -60,7 +60,7 @@
Output: default/dest1
query: SELECT dest1.* FROM dest1
Input: default/dest1
-Output: /data/users/athusoo/commits/hive_trunk_ws8/ql/../build/ql/tmp/627000844/1811724340.10000
+Output: /data/users/zshao/tools/495-trunk-apache-hive/ql/../build/ql/tmp/108328031/145804882.10000
3 1 1
3 1 1
3 1 1
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath3.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath3.q.out?rev=781121&r1=781120&r2=781121&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath3.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath3.q.out Tue Jun 2 18:51:20 2009
@@ -18,7 +18,7 @@
expr: mstringstring
type: map<string,string>
expr: lintstring
- type: array<struct<myint:int,mystring:string>>
+ type: array<struct<myint:int,mystring:string,underscore_int:int>>
Select Operator
expressions:
expr: 0['key_9']
@@ -40,7 +40,7 @@
query: FROM src_thrift
SELECT src_thrift.mstringstring['key_9'], src_thrift.lintstring.myint
Input: default/src_thrift
-Output: /data/users/athusoo/commits/hive_trunk_ws8/ql/../build/ql/tmp/535177843/787258899.10000
+Output: /data/users/zshao/tools/495-trunk-apache-hive/ql/../build/ql/tmp/47313672/383253534.10000
NULL [0]
NULL [1]
NULL [4]
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath4.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath4.q.out?rev=781121&r1=781120&r2=781121&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath4.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath4.q.out Tue Jun 2 18:51:20 2009
@@ -21,7 +21,7 @@
expr: mstringstring
type: map<string,string>
expr: lintstring
- type: array<struct<myint:int,mystring:string>>
+ type: array<struct<myint:int,mystring:string,underscore_int:int>>
Filter Operator
predicate:
expr: ((0['key_9'] is not null and 1.myint is not null) and 1 is not null)
@@ -50,7 +50,7 @@
OR lintstring.myint IS NOT NULL
OR lintstring IS NOT NULL
Input: default/src_thrift
-Output: /data/users/athusoo/commits/hive_trunk_ws8/ql/../build/ql/tmp/9314654/120165410.10000
+Output: /data/users/zshao/tools/495-trunk-apache-hive/ql/../build/ql/tmp/115422682/57144029.10000
NULL [0]
NULL [1]
NULL [4]
Added: hadoop/hive/trunk/ql/src/test/results/clientpositive/join_thrift.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/join_thrift.q.out?rev=781121&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/join_thrift.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/join_thrift.q.out Tue Jun 2 18:51:20 2009
@@ -0,0 +1,103 @@
+query: DESCRIBE src_thrift
+aint int from deserializer
+astring string from deserializer
+lint array<int> from deserializer
+lstring array<string> from deserializer
+lintstring array<org.apache.hadoop.hive.serde2.thrift.test.IntString> from deserializer
+mstringstring map<string,string> from deserializer
+query: EXPLAIN
+SELECT s1.aint, s2.lintstring
+FROM src_thrift s1
+JOIN src_thrift s2
+ON s1.aint = s2.aint
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF src_thrift s1) (TOK_TABREF src_thrift s2) (= (. (TOK_TABLE_OR_COL s1) aint) (. (TOK_TABLE_OR_COL s2) aint)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL s1) aint)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL s2) lintstring)))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ s2
+ Select Operator
+ expressions:
+ expr: aint
+ type: int
+ expr: lintstring
+ type: array<struct<myint:int,mystring:string,underscore_int:int>>
+ Reduce Output Operator
+ key expressions:
+ expr: 0
+ type: int
+ sort order: +
+ Map-reduce partition columns:
+ expr: 0
+ type: int
+ tag: 1
+ value expressions:
+ expr: 0
+ type: int
+ expr: 1
+ type: array<struct<myint:int,mystring:string,underscore_int:int>>
+ s1
+ Select Operator
+ expressions:
+ expr: aint
+ type: int
+ Reduce Output Operator
+ key expressions:
+ expr: 0
+ type: int
+ sort order: +
+ Map-reduce partition columns:
+ expr: 0
+ type: int
+ tag: 0
+ value expressions:
+ expr: 0
+ type: int
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE.0}
+ 1 {VALUE.0} {VALUE.1}
+ Select Operator
+ expressions:
+ expr: 0
+ type: int
+ expr: 2
+ type: array<struct<myint:int,mystring:string,underscore_int:int>>
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+query: SELECT s1.aint, s2.lintstring
+FROM src_thrift s1
+JOIN src_thrift s2
+ON s1.aint = s2.aint
+Input: default/src_thrift
+Output: /data/users/zshao/tools/495-trunk-apache-hive/ql/../build/ql/tmp/1611059524/151199080.10000
+-1952710710 [{"myint":25,"mystring":"125","underscore_int":5}]
+-1461153973 [{"myint":49,"mystring":"343","underscore_int":7}]
+-751827638 [{"myint":4,"mystring":"8","underscore_int":2}]
+-734328909 [{"myint":16,"mystring":"64","underscore_int":4}]
+0 null
+336964413 [{"myint":81,"mystring":"729","underscore_int":9}]
+465985200 [{"myint":1,"mystring":"1","underscore_int":1}]
+477111222 [{"myint":9,"mystring":"27","underscore_int":3}]
+1244525190 [{"myint":36,"mystring":"216","underscore_int":6}]
+1638581578 [{"myint":64,"mystring":"512","underscore_int":8}]
+1712634731 [{"myint":0,"mystring":"0","underscore_int":0}]
Modified: hadoop/hive/trunk/serde/if/test/complex.thrift
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/if/test/complex.thrift?rev=781121&r1=781120&r2=781121&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/if/test/complex.thrift (original)
+++ hadoop/hive/trunk/serde/if/test/complex.thrift Tue Jun 2 18:51:20 2009
@@ -3,6 +3,7 @@
struct IntString {
1: i32 myint;
2: string myString;
+ 3: i32 underscore_int;
}
struct Complex {
Modified: hadoop/hive/trunk/serde/src/gen-java/org/apache/hadoop/hive/serde2/thrift/test/IntString.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/gen-java/org/apache/hadoop/hive/serde2/thrift/test/IntString.java?rev=781121&r1=781120&r2=781121&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/gen-java/org/apache/hadoop/hive/serde2/thrift/test/IntString.java (original)
+++ hadoop/hive/trunk/serde/src/gen-java/org/apache/hadoop/hive/serde2/thrift/test/IntString.java Tue Jun 2 18:51:20 2009
@@ -19,11 +19,13 @@
public class IntString implements TBase, java.io.Serializable {
public int myint;
public String myString;
+ public int underscore_int;
public final Isset __isset = new Isset();
public static final class Isset implements java.io.Serializable {
public boolean myint = false;
public boolean myString = false;
+ public boolean underscore_int = false;
}
public IntString() {
@@ -31,13 +33,16 @@
public IntString(
int myint,
- String myString)
+ String myString,
+ int underscore_int)
{
this();
this.myint = myint;
this.__isset.myint = true;
this.myString = myString;
this.__isset.myString = true;
+ this.underscore_int = underscore_int;
+ this.__isset.underscore_int = true;
}
public boolean equals(Object that) {
@@ -70,6 +75,15 @@
return false;
}
+ boolean this_present_underscore_int = true;
+ boolean that_present_underscore_int = true;
+ if (this_present_underscore_int || that_present_underscore_int) {
+ if (!(this_present_underscore_int && that_present_underscore_int))
+ return false;
+ if (this.underscore_int != that.underscore_int)
+ return false;
+ }
+
return true;
}
@@ -104,6 +118,14 @@
TProtocolUtil.skip(iprot, field.type);
}
break;
+ case 3:
+ if (field.type == TType.I32) {
+ this.underscore_int = iprot.readI32();
+ this.__isset.underscore_int = true;
+ } else {
+ TProtocolUtil.skip(iprot, field.type);
+ }
+ break;
default:
TProtocolUtil.skip(iprot, field.type);
break;
@@ -131,6 +153,12 @@
oprot.writeString(this.myString);
oprot.writeFieldEnd();
}
+ field.name = "underscore_int";
+ field.type = TType.I32;
+ field.id = 3;
+ oprot.writeFieldBegin(field);
+ oprot.writeI32(this.underscore_int);
+ oprot.writeFieldEnd();
oprot.writeFieldStop();
oprot.writeStructEnd();
}
@@ -141,6 +169,8 @@
sb.append(this.myint);
sb.append(",myString:");
sb.append(this.myString);
+ sb.append(",underscore_int:");
+ sb.append(this.underscore_int);
sb.append(")");
return sb.toString();
}
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java?rev=781121&r1=781120&r2=781121&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java Tue Jun 2 18:51:20 2009
@@ -113,7 +113,7 @@
}
/**
- * Get the standard ObjectInspector for an ObjectInspector.
+ * Get the corresponding standard ObjectInspector for an ObjectInspector.
*
* The returned ObjectInspector can be used to inspect the standard object.
*/
@@ -144,14 +144,15 @@
}
case LIST: {
ListObjectInspector loi = (ListObjectInspector)oi;
- result = ObjectInspectorFactory.getStandardListObjectInspector(loi.getListElementObjectInspector());
+ result = ObjectInspectorFactory.getStandardListObjectInspector(
+ getStandardObjectInspector(loi.getListElementObjectInspector(), objectInspectorOption));
break;
}
case MAP: {
MapObjectInspector moi = (MapObjectInspector)oi;
result = ObjectInspectorFactory.getStandardMapObjectInspector(
- moi.getMapKeyObjectInspector(),
- moi.getMapValueObjectInspector());
+ getStandardObjectInspector(moi.getMapKeyObjectInspector(), objectInspectorOption),
+ getStandardObjectInspector(moi.getMapValueObjectInspector(), objectInspectorOption));
break;
}
case STRUCT: {
@@ -161,7 +162,7 @@
List<ObjectInspector> fieldObjectInspectors = new ArrayList<ObjectInspector>(fields.size());
for(StructField f : fields) {
fieldNames.add(f.getFieldName());
- fieldObjectInspectors.add(f.getFieldObjectInspector());
+ fieldObjectInspectors.add(getStandardObjectInspector(f.getFieldObjectInspector(), objectInspectorOption));
}
result = ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldObjectInspectors);
break;
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java?rev=781121&r1=781120&r2=781121&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java Tue Jun 2 18:51:20 2009
@@ -59,16 +59,20 @@
private static class Token {
public int position;
public String text;
- public boolean isAlphaDigit;
+ public boolean isType;
public String toString() {
return "" + position + ":" + text;
}
};
+ private static boolean isTypeChar(char c) {
+ return Character.isLetterOrDigit(c) || c == '_' || c == '.';
+ }
+
/**
* Tokenize the typeInfoString.
- * The rule is simple: all consecutive alphadigits are in one token, and all
- * other characters are one character per token.
+ * The rule is simple: all consecutive alphadigits and '_', '.' are in one
+ * token, and all other characters are one character per token.
*
* tokenize("map<int,string>") should return ["map","<","int",",","string",">"]
*/
@@ -79,12 +83,12 @@
while (end <= typeInfoString.length()) {
// last character ends a token?
if (end == typeInfoString.length()
- || !Character.isLetterOrDigit(typeInfoString.charAt(end-1))
- || !Character.isLetterOrDigit(typeInfoString.charAt(end))) {
+ || !isTypeChar(typeInfoString.charAt(end-1))
+ || !isTypeChar(typeInfoString.charAt(end))) {
Token t = new Token();
t.position = begin;
t.text = typeInfoString.substring(begin, end);
- t.isAlphaDigit = Character.isLetterOrDigit(typeInfoString.charAt(begin));
+ t.isType = isTypeChar(typeInfoString.charAt(begin));
tokens.add(t);
begin = end;
}
@@ -138,17 +142,17 @@
&& null == PrimitiveObjectInspectorUtils.getTypeEntryFromTypeName(t.text)
&& !t.text.equals(alternative)) {
throw new IllegalArgumentException("Error: " + item + " expected at the position "
- + t.position + " of '" + typeInfoString + "'" );
+ + t.position + " of '" + typeInfoString + "' but '" + t.text + "' is found." );
}
} else if (item.equals("name")) {
- if (!t.isAlphaDigit && !t.text.equals(alternative)) {
+ if (!t.isType && !t.text.equals(alternative)) {
throw new IllegalArgumentException("Error: " + item + " expected at the position "
- + t.position + " of '" + typeInfoString + "'" );
+ + t.position + " of '" + typeInfoString + "' but '" + t.text + "' is found." );
}
} else {
if (!item.equals(t.text) && !t.text.equals(alternative)) {
throw new IllegalArgumentException("Error: " + item + " expected at the position "
- + t.position + " of '" + typeInfoString + "'" );
+ + t.position + " of '" + typeInfoString + "' but '" + t.text + "' is found." );
}
}
iToken ++;
Modified: hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorUtils.java?rev=781121&r1=781120&r2=781121&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorUtils.java (original)
+++ hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorUtils.java Tue Jun 2 18:51:20 2009
@@ -85,9 +85,10 @@
assertEquals(ObjectInspectorFactory.getStandardListObjectInspector(
PrimitiveObjectInspectorFactory.javaStringObjectInspector),
fields.get(3).getFieldObjectInspector());
- assertEquals(ObjectInspectorFactory.getStandardListObjectInspector(
- ObjectInspectorFactory.getReflectionObjectInspector(IntString.class,
- ObjectInspectorFactory.ObjectInspectorOptions.THRIFT)),
+ assertEquals(ObjectInspectorUtils.getStandardObjectInspector(
+ ObjectInspectorFactory.getStandardListObjectInspector(
+ ObjectInspectorFactory.getReflectionObjectInspector(IntString.class,
+ ObjectInspectorFactory.ObjectInspectorOptions.THRIFT))),
fields.get(4).getFieldObjectInspector());
assertEquals(ObjectInspectorFactory.getStandardMapObjectInspector(
PrimitiveObjectInspectorFactory.javaStringObjectInspector,
Modified: hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/thrift_test/CreateSequenceFile.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/thrift_test/CreateSequenceFile.java?rev=781121&r1=781120&r2=781121&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/thrift_test/CreateSequenceFile.java (original)
+++ hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/thrift_test/CreateSequenceFile.java Tue Jun 2 18:51:20 2009
@@ -106,7 +106,7 @@
ArrayList<String> slist = new ArrayList<String>();
slist.add("" + i*10); slist.add("" + i*100); slist.add("" + i*1000);
ArrayList<IntString> islist = new ArrayList<IntString>();
- islist.add(new IntString(i*i, ""+ i*i*i));
+ islist.add(new IntString(i*i, ""+ i*i*i, i));
HashMap<String,String> hash = new HashMap<String,String>();
hash.put("key_" + i, "value_" + i);