You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by zs...@apache.org on 2009/02/17 20:39:32 UTC
svn commit: r745212 [8/9] - in /hadoop/hive/trunk: ./ metastore/if/
metastore/src/gen-javabean/org/apache/hadoop/hive/metastore/api/
metastore/src/gen-php/ metastore/src/gen-py/hive_metastore/
metastore/src/java/org/apache/hadoop/hive/metastore/ ql/src...
Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample4.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample4.q.xml?rev=745212&r1=745211&r2=745212&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample4.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample4.q.xml Tue Feb 17 19:39:29 2009
@@ -31,7 +31,7 @@
<boolean>true</boolean>
</void>
<void property="sourceDir">
- <string>/tmp/hive-njain/1853542808/45744403.10000.insclause-0</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-270/ql/../build/ql/tmp/17506868/803084486.10000.insclause-0</string>
</void>
<void property="table">
<object id="tableDesc0" class="org.apache.hadoop.hive.ql.plan.tableDesc">
@@ -51,6 +51,10 @@
<string>dest1</string>
</void>
<void method="put">
+ <string>columns.types</string>
+ <string>string:string</string>
+ </void>
+ <void method="put">
<string>serialization.ddl</string>
<string>struct dest1 { string key, string value}</string>
</void>
@@ -80,7 +84,7 @@
</void>
<void method="put">
<string>location</string>
- <string>file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest1</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/dest1</string>
</void>
</object>
</void>
@@ -120,7 +124,7 @@
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc">
<void property="dirName">
- <string>/tmp/hive-njain/1853542808/45744403.10000.insclause-0</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-270/ql/../build/ql/tmp/17506868/803084486.10000.insclause-0</string>
</void>
<void property="tableInfo">
<object idref="tableDesc0"/>
@@ -254,7 +258,7 @@
<void property="pathToAliases">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcbucket/kv1.txt</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/srcbucket/kv1.txt</string>
<object class="java.util.ArrayList">
<void method="add">
<string>s</string>
@@ -266,7 +270,7 @@
<void property="pathToPartitionInfo">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcbucket/kv1.txt</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/srcbucket/kv1.txt</string>
<object class="org.apache.hadoop.hive.ql.plan.partitionDesc">
<void property="partSpec">
<object class="java.util.LinkedHashMap"/>
@@ -289,6 +293,10 @@
<string>srcbucket</string>
</void>
<void method="put">
+ <string>columns.types</string>
+ <string>string:string</string>
+ </void>
+ <void method="put">
<string>bucket_field_name</string>
<string>key</string>
</void>
@@ -322,7 +330,7 @@
</void>
<void method="put">
<string>location</string>
- <string>file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcbucket</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/srcbucket</string>
</void>
</object>
</void>
Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample5.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample5.q.xml?rev=745212&r1=745211&r2=745212&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample5.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample5.q.xml Tue Feb 17 19:39:29 2009
@@ -31,7 +31,7 @@
<boolean>true</boolean>
</void>
<void property="sourceDir">
- <string>/tmp/hive-njain/824806517/591440948.10000.insclause-0</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-270/ql/../build/ql/tmp/433822082/971124421.10000.insclause-0</string>
</void>
<void property="table">
<object id="tableDesc0" class="org.apache.hadoop.hive.ql.plan.tableDesc">
@@ -51,6 +51,10 @@
<string>dest1</string>
</void>
<void method="put">
+ <string>columns.types</string>
+ <string>string:string</string>
+ </void>
+ <void method="put">
<string>serialization.ddl</string>
<string>struct dest1 { string key, string value}</string>
</void>
@@ -80,7 +84,7 @@
</void>
<void method="put">
<string>location</string>
- <string>file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest1</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/dest1</string>
</void>
</object>
</void>
@@ -124,7 +128,7 @@
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc">
<void property="dirName">
- <string>/tmp/hive-njain/824806517/591440948.10000.insclause-0</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-270/ql/../build/ql/tmp/433822082/971124421.10000.insclause-0</string>
</void>
<void property="tableInfo">
<object idref="tableDesc0"/>
@@ -426,7 +430,7 @@
<void property="pathToAliases">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcbucket</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/srcbucket</string>
<object class="java.util.ArrayList">
<void method="add">
<string>s</string>
@@ -438,7 +442,7 @@
<void property="pathToPartitionInfo">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcbucket</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/srcbucket</string>
<object class="org.apache.hadoop.hive.ql.plan.partitionDesc">
<void property="partSpec">
<object class="java.util.LinkedHashMap"/>
@@ -461,6 +465,10 @@
<string>srcbucket</string>
</void>
<void method="put">
+ <string>columns.types</string>
+ <string>string:string</string>
+ </void>
+ <void method="put">
<string>bucket_field_name</string>
<string>key</string>
</void>
@@ -494,7 +502,7 @@
</void>
<void method="put">
<string>location</string>
- <string>file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcbucket</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/srcbucket</string>
</void>
</object>
</void>
Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample6.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample6.q.xml?rev=745212&r1=745211&r2=745212&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample6.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample6.q.xml Tue Feb 17 19:39:29 2009
@@ -31,7 +31,7 @@
<boolean>true</boolean>
</void>
<void property="sourceDir">
- <string>/tmp/hive-njain/814020855/624762724.10000.insclause-0</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-270/ql/../build/ql/tmp/59637097/284842510.10000.insclause-0</string>
</void>
<void property="table">
<object id="tableDesc0" class="org.apache.hadoop.hive.ql.plan.tableDesc">
@@ -51,6 +51,10 @@
<string>dest1</string>
</void>
<void method="put">
+ <string>columns.types</string>
+ <string>string:string</string>
+ </void>
+ <void method="put">
<string>serialization.ddl</string>
<string>struct dest1 { string key, string value}</string>
</void>
@@ -80,7 +84,7 @@
</void>
<void method="put">
<string>location</string>
- <string>file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest1</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/dest1</string>
</void>
</object>
</void>
@@ -124,7 +128,7 @@
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc">
<void property="dirName">
- <string>/tmp/hive-njain/814020855/624762724.10000.insclause-0</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-270/ql/../build/ql/tmp/59637097/284842510.10000.insclause-0</string>
</void>
<void property="tableInfo">
<object idref="tableDesc0"/>
@@ -426,7 +430,7 @@
<void property="pathToAliases">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcbucket/kv1.txt</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/srcbucket/kv1.txt</string>
<object class="java.util.ArrayList">
<void method="add">
<string>s</string>
@@ -438,7 +442,7 @@
<void property="pathToPartitionInfo">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcbucket/kv1.txt</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/srcbucket/kv1.txt</string>
<object class="org.apache.hadoop.hive.ql.plan.partitionDesc">
<void property="partSpec">
<object class="java.util.LinkedHashMap"/>
@@ -461,6 +465,10 @@
<string>srcbucket</string>
</void>
<void method="put">
+ <string>columns.types</string>
+ <string>string:string</string>
+ </void>
+ <void method="put">
<string>bucket_field_name</string>
<string>key</string>
</void>
@@ -494,7 +502,7 @@
</void>
<void method="put">
<string>location</string>
- <string>file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/srcbucket</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/srcbucket</string>
</void>
</object>
</void>
Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample7.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample7.q.xml?rev=745212&r1=745211&r2=745212&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample7.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/sample7.q.xml Tue Feb 17 19:39:29 2009
@@ -31,7 +31,7 @@
<boolean>true</boolean>
</void>
<void property="sourceDir">
- <string>/data/users/athusoo/apacheprojects/hive_local_ws3/ql/../build/ql/tmp/1538040054/309610911.10000.insclause-0</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-270/ql/../build/ql/tmp/709018763/557734057.10000.insclause-0</string>
</void>
<void property="table">
<object id="tableDesc0" class="org.apache.hadoop.hive.ql.plan.tableDesc">
@@ -51,6 +51,10 @@
<string>dest1</string>
</void>
<void method="put">
+ <string>columns.types</string>
+ <string>string:string</string>
+ </void>
+ <void method="put">
<string>serialization.ddl</string>
<string>struct dest1 { string key, string value}</string>
</void>
@@ -80,7 +84,7 @@
</void>
<void method="put">
<string>location</string>
- <string>file:/data/users/athusoo/apacheprojects/hive_local_ws3/build/ql/test/data/warehouse/dest1</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/dest1</string>
</void>
</object>
</void>
@@ -128,7 +132,7 @@
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc">
<void property="dirName">
- <string>/data/users/athusoo/apacheprojects/hive_local_ws3/ql/../build/ql/tmp/1538040054/309610911.10000.insclause-0</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-270/ql/../build/ql/tmp/709018763/557734057.10000.insclause-0</string>
</void>
<void property="tableInfo">
<object idref="tableDesc0"/>
@@ -551,7 +555,7 @@
<void property="pathToAliases">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/data/users/athusoo/apacheprojects/hive_local_ws3/build/ql/test/data/warehouse/srcbucket/kv1.txt</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/srcbucket/kv1.txt</string>
<object class="java.util.ArrayList">
<void method="add">
<string>s</string>
@@ -563,7 +567,7 @@
<void property="pathToPartitionInfo">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/data/users/athusoo/apacheprojects/hive_local_ws3/build/ql/test/data/warehouse/srcbucket/kv1.txt</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/srcbucket/kv1.txt</string>
<object class="org.apache.hadoop.hive.ql.plan.partitionDesc">
<void property="partSpec">
<object class="java.util.LinkedHashMap"/>
@@ -586,6 +590,10 @@
<string>srcbucket</string>
</void>
<void method="put">
+ <string>columns.types</string>
+ <string>string:string</string>
+ </void>
+ <void method="put">
<string>bucket_field_name</string>
<string>key</string>
</void>
@@ -619,7 +627,7 @@
</void>
<void method="put">
<string>location</string>
- <string>file:/data/users/athusoo/apacheprojects/hive_local_ws3/build/ql/test/data/warehouse/srcbucket</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/srcbucket</string>
</void>
</object>
</void>
Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/subq.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/subq.q.xml?rev=745212&r1=745211&r2=745212&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/subq.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/subq.q.xml Tue Feb 17 19:39:29 2009
@@ -28,7 +28,7 @@
<boolean>true</boolean>
</void>
<void property="sourceDir">
- <string>/data/users/athusoo/apacheprojects/hive_local_ws3/ql/../build/ql/tmp/538025002/104579387.10000.insclause-0</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-270/ql/../build/ql/tmp/51449299/1751648352.10000.insclause-0</string>
</void>
<void property="targetDir">
<string>../build/ql/test/data/warehouse/union.out</string>
@@ -75,7 +75,7 @@
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc">
<void property="dirName">
- <string>/data/users/athusoo/apacheprojects/hive_local_ws3/ql/../build/ql/tmp/538025002/104579387.10000.insclause-0</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-270/ql/../build/ql/tmp/51449299/1751648352.10000.insclause-0</string>
</void>
<void property="tableInfo">
<object class="org.apache.hadoop.hive.ql.plan.tableDesc">
@@ -430,7 +430,7 @@
<void property="pathToAliases">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/data/users/athusoo/apacheprojects/hive_local_ws3/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/src</string>
<object class="java.util.ArrayList">
<void method="add">
<string>unioninput:src</string>
@@ -442,7 +442,7 @@
<void property="pathToPartitionInfo">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/data/users/athusoo/apacheprojects/hive_local_ws3/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/src</string>
<object class="org.apache.hadoop.hive.ql.plan.partitionDesc">
<void property="partSpec">
<object class="java.util.LinkedHashMap"/>
@@ -465,6 +465,10 @@
<string>src</string>
</void>
<void method="put">
+ <string>columns.types</string>
+ <string>string:string</string>
+ </void>
+ <void method="put">
<string>serialization.ddl</string>
<string>struct src { string key, string value}</string>
</void>
@@ -494,7 +498,7 @@
</void>
<void method="put">
<string>location</string>
- <string>file:/data/users/athusoo/apacheprojects/hive_local_ws3/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/src</string>
</void>
</object>
</void>
Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/udf1.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/udf1.q.xml?rev=745212&r1=745211&r2=745212&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/udf1.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/udf1.q.xml Tue Feb 17 19:39:29 2009
@@ -30,7 +30,7 @@
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc">
<void property="dirName">
- <string>/data/users/athusoo/apacheprojects/hive_local_ws3/ql/../build/ql/tmp/1023952720.10001.insclause-0</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-270/ql/../build/ql/tmp/1309846302.10001.insclause-0</string>
</void>
<void property="tableInfo">
<object class="org.apache.hadoop.hive.ql.plan.tableDesc">
@@ -1340,7 +1340,7 @@
<void property="pathToAliases">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/data/users/athusoo/apacheprojects/hive_local_ws3/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/src</string>
<object class="java.util.ArrayList">
<void method="add">
<string>src</string>
@@ -1352,7 +1352,7 @@
<void property="pathToPartitionInfo">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/data/users/athusoo/apacheprojects/hive_local_ws3/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/src</string>
<object class="org.apache.hadoop.hive.ql.plan.partitionDesc">
<void property="partSpec">
<object class="java.util.LinkedHashMap"/>
@@ -1375,6 +1375,10 @@
<string>src</string>
</void>
<void method="put">
+ <string>columns.types</string>
+ <string>string:string</string>
+ </void>
+ <void method="put">
<string>serialization.ddl</string>
<string>struct src { string key, string value}</string>
</void>
@@ -1404,7 +1408,7 @@
</void>
<void method="put">
<string>location</string>
- <string>file:/data/users/athusoo/apacheprojects/hive_local_ws3/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/src</string>
</void>
</object>
</void>
Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/udf4.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/udf4.q.xml?rev=745212&r1=745211&r2=745212&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/udf4.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/udf4.q.xml Tue Feb 17 19:39:29 2009
@@ -26,7 +26,7 @@
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc">
<void property="dirName">
- <string>/data/users/zshao/sync/pp/ql/../build/ql/tmp/12573127.10001.insclause-0</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-270/ql/../build/ql/tmp/1343185370.10001.insclause-0</string>
</void>
<void property="tableInfo">
<object class="org.apache.hadoop.hive.ql.plan.tableDesc">
@@ -1154,7 +1154,7 @@
<void property="pathToAliases">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/data/users/zshao/sync/pp/build/ql/test/data/warehouse/dest1</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/dest1</string>
<object class="java.util.ArrayList">
<void method="add">
<string>dest1</string>
@@ -1166,7 +1166,7 @@
<void property="pathToPartitionInfo">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/data/users/zshao/sync/pp/build/ql/test/data/warehouse/dest1</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/dest1</string>
<object class="org.apache.hadoop.hive.ql.plan.partitionDesc">
<void property="partSpec">
<object class="java.util.LinkedHashMap"/>
@@ -1189,6 +1189,10 @@
<string>dest1</string>
</void>
<void method="put">
+ <string>columns.types</string>
+ <string>string:string</string>
+ </void>
+ <void method="put">
<string>serialization.ddl</string>
<string>struct dest1 { string key, string value}</string>
</void>
@@ -1218,7 +1222,7 @@
</void>
<void method="put">
<string>location</string>
- <string>file:/data/users/zshao/sync/pp/build/ql/test/data/warehouse/dest1</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/dest1</string>
</void>
</object>
</void>
Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/udf6.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/udf6.q.xml?rev=745212&r1=745211&r2=745212&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/udf6.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/udf6.q.xml Tue Feb 17 19:39:29 2009
@@ -26,7 +26,7 @@
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc">
<void property="dirName">
- <string>/data/users/zshao/sync/apache-trunk-HIVE-164/ql/../build/ql/tmp/323777374.10001.insclause-0</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-270/ql/../build/ql/tmp/343375627.10001.insclause-0</string>
</void>
<void property="tableInfo">
<object class="org.apache.hadoop.hive.ql.plan.tableDesc">
@@ -295,7 +295,7 @@
<void property="pathToAliases">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/data/users/zshao/sync/apache-trunk-HIVE-164/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/src</string>
<object class="java.util.ArrayList">
<void method="add">
<string>src</string>
@@ -307,7 +307,7 @@
<void property="pathToPartitionInfo">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/data/users/zshao/sync/apache-trunk-HIVE-164/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/src</string>
<object class="org.apache.hadoop.hive.ql.plan.partitionDesc">
<void property="partSpec">
<object class="java.util.LinkedHashMap"/>
@@ -330,6 +330,10 @@
<string>src</string>
</void>
<void method="put">
+ <string>columns.types</string>
+ <string>string:string</string>
+ </void>
+ <void method="put">
<string>serialization.ddl</string>
<string>struct src { string key, string value}</string>
</void>
@@ -359,7 +363,7 @@
</void>
<void method="put">
<string>location</string>
- <string>file:/data/users/zshao/sync/apache-trunk-HIVE-164/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/src</string>
</void>
</object>
</void>
Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/union.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/union.q.xml?rev=745212&r1=745211&r2=745212&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/union.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/union.q.xml Tue Feb 17 19:39:29 2009
@@ -28,7 +28,7 @@
<boolean>true</boolean>
</void>
<void property="sourceDir">
- <string>/data/users/athusoo/apacheprojects/hive_local_ws3/ql/../build/ql/tmp/323358384/1524460804.10000.insclause-0</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-270/ql/../build/ql/tmp/449888396/1582541203.10000.insclause-0</string>
</void>
<void property="targetDir">
<string>../build/ql/test/data/warehouse/union.out</string>
@@ -79,7 +79,7 @@
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc">
<void property="dirName">
- <string>/data/users/athusoo/apacheprojects/hive_local_ws3/ql/../build/ql/tmp/323358384/1524460804.10000.insclause-0</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-270/ql/../build/ql/tmp/449888396/1582541203.10000.insclause-0</string>
</void>
<void property="tableInfo">
<object class="org.apache.hadoop.hive.ql.plan.tableDesc">
@@ -690,7 +690,7 @@
<void property="pathToAliases">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/data/users/athusoo/apacheprojects/hive_local_ws3/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/src</string>
<object class="java.util.ArrayList">
<void method="add">
<string>null-subquery1:unioninput-subquery1:src</string>
@@ -705,7 +705,7 @@
<void property="pathToPartitionInfo">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/data/users/athusoo/apacheprojects/hive_local_ws3/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/src</string>
<object class="org.apache.hadoop.hive.ql.plan.partitionDesc">
<void property="partSpec">
<object class="java.util.LinkedHashMap"/>
@@ -728,6 +728,10 @@
<string>src</string>
</void>
<void method="put">
+ <string>columns.types</string>
+ <string>string:string</string>
+ </void>
+ <void method="put">
<string>serialization.ddl</string>
<string>struct src { string key, string value}</string>
</void>
@@ -757,7 +761,7 @@
</void>
<void method="put">
<string>location</string>
- <string>file:/data/users/athusoo/apacheprojects/hive_local_ws3/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/src</string>
</void>
</object>
</void>
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java?rev=745212&r1=745211&r2=745212&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java Tue Feb 17 19:39:29 2009
@@ -70,12 +70,12 @@
nativeSerDeNames.add(org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe.class.getName());
// For backward compatibility
nativeSerDeNames.add("org.apache.hadoop.hive.serde.thrift.columnsetSerDe");
+ nativeSerDeNames.add(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName());
}
public static boolean isNativeSerDe(String serde) {
return nativeSerDeNames.contains(serde);
}
-
private static boolean initCoreSerDes = registerCoreSerDes();
Added: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyByte.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyByte.java?rev=745212&view=auto
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyByte.java (added)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyByte.java Tue Feb 17 19:39:29 2009
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.lazy;
+
+/**
+ * LazyObject for storing a value of Byte.
+ *
+ * <p>
+ * Part of the code is adapted from Apache Harmony Project.
+ *
+ * As with the specification, this implementation relied on code laid out in <a
+ * href="http://www.hackersdelight.org/">Henry S. Warren, Jr.'s Hacker's
+ * Delight, (Addison Wesley, 2002)</a> as well as <a
+ * href="http://aggregate.org/MAGIC/">The Aggregate's Magic Algorithms</a>.
+ * </p>
+ *
+ */
+public class LazyByte extends LazyPrimitive<Byte> {
+
+ public LazyByte() {
+ super(Byte.class);
+ }
+
+ @Override
+ public Byte getPrimitiveObject() {
+ if (bytes == null) return null;
+ try {
+ // Slower method: convert to String and then convert to Integer
+ // return Byte.valueOf(LazyUtils.convertToString(bytes, start, length));
+ return Byte.valueOf(parseByte(bytes, start, length));
+ } catch (NumberFormatException e) {
+ return null;
+ }
+ }
+
+
+ /**
+ * Parses the string argument as if it was a byte value and returns the
+ * result. Throws NumberFormatException if the string does not represent a
+ * single byte quantity.
+ *
+ * @param bytes
+ * @param start
+ * @param length
+ * a UTF-8 encoded string representation of a single byte quantity.
+ * @return byte the value represented by the argument
+ * @throws NumberFormatException
+ * if the argument could not be parsed as a byte quantity.
+ */
+ public static byte parseByte(byte[] bytes, int start, int length) throws NumberFormatException {
+ return parseByte(bytes, start, length, 10);
+ }
+
+ /**
+ * Parses the string argument as if it was a byte value and returns the
+ * result. Throws NumberFormatException if the string does not represent a
+ * single byte quantity. The second argument specifies the radix to use when
+ * parsing the value.
+ *
+ * @param bytes
+ * @param start
+ * @param length
+ * a UTF-8 encoded string representation of a single byte quantity.
+ * @param radix
+ * the radix to use when parsing.
+ * @return byte the value represented by the argument
+ * @throws NumberFormatException
+ * if the argument could not be parsed as a byte quantity.
+ */
+ public static byte parseByte(byte[] bytes, int start, int length, int radix)
+ throws NumberFormatException {
+ int intValue = LazyInteger.parseInt(bytes, start, length, radix);
+ byte result = (byte) intValue;
+ if (result == intValue) {
+ return result;
+ }
+ throw new NumberFormatException();
+ }
+
+}
Added: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDouble.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDouble.java?rev=745212&view=auto
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDouble.java (added)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDouble.java Tue Feb 17 19:39:29 2009
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.lazy;
+
+import java.nio.charset.CharacterCodingException;
+
+import org.apache.hadoop.io.Text;
+
+/**
+ * LazyObject for storing a value of Double.
+ *
+ */
+public class LazyDouble extends LazyPrimitive<Double> {
+
+ public LazyDouble() {
+ super(Double.class);
+ }
+
+ Text text = new Text();
+
+ @Override
+ public Double getPrimitiveObject() {
+ // TODO: replace this by directly parsing the bytes buffer for better performance.
+ if (bytes == null) return null;
+ try {
+ return Double.valueOf(Text.decode(bytes, start, length));
+ } catch (NumberFormatException e) {
+ return null;
+ } catch (CharacterCodingException e) {
+ return null;
+ }
+ }
+
+}
Added: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java?rev=745212&view=auto
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java (added)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java Tue Feb 17 19:39:29 2009
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.lazy;
+
+/**
+ * LazyObject for storing a value of Integer.
+ *
+ * <p>
+ * Part of the code is adapted from Apache Harmony Project.
+ *
+ * As with the specification, this implementation relied on code laid out in <a
+ * href="http://www.hackersdelight.org/">Henry S. Warren, Jr.'s Hacker's
+ * Delight, (Addison Wesley, 2002)</a> as well as <a
+ * href="http://aggregate.org/MAGIC/">The Aggregate's Magic Algorithms</a>.
+ * </p>
+ *
+ */
+public class LazyInteger extends LazyPrimitive<Integer> {
+
+ public LazyInteger() {
+ super(Integer.class);
+ }
+
+ @Override
+ public Integer getPrimitiveObject() {
+ try {
+ // Slower method: convert to String and then convert to Integer
+ // return Integer.valueOf(LazyUtils.convertToString(bytes, start, length));
+ return Integer.valueOf(parseInt(bytes, start, length));
+ } catch (NumberFormatException e) {
+ return null;
+ }
+ }
+
+ /**
+ * Parses the string argument as if it was an int value and returns the
+ * result. Throws NumberFormatException if the string does not represent an
+ * int quantity.
+ *
+ * @param bytes
+ * @param start
+ * @param length
+ * a UTF-8 encoded string representation of an int quantity.
+ * @return int the value represented by the argument
+ * @exception NumberFormatException
+ * if the argument could not be parsed as an int quantity.
+ */
+ public static int parseInt(byte[] bytes, int start, int length) throws NumberFormatException {
+ return parseInt(bytes, start, length, 10);
+ }
+
+ /**
+ * Parses the string argument as if it was an int value and returns the
+ * result. Throws NumberFormatException if the string does not represent an
+ * int quantity. The second argument specifies the radix to use when parsing
+ * the value.
+ *
+ * @param bytes
+ * @param start
+ * @param length
+ * a UTF-8 encoded string representation of an int quantity.
+ * @param radix
+ * the base to use for conversion.
+ * @return int the value represented by the argument
+ * @exception NumberFormatException
+ * if the argument could not be parsed as an int quantity.
+ */
+ public static int parseInt(byte[] bytes, int start, int length, int radix)
+ throws NumberFormatException {
+ if (bytes == null) {
+ throw new NumberFormatException("String is null");
+ }
+ if (radix < Character.MIN_RADIX ||
+ radix > Character.MAX_RADIX) {
+ throw new NumberFormatException("Invalid radix: " + radix);
+ }
+ if (length == 0) {
+ throw new NumberFormatException("Empty string!");
+ }
+ int offset = start;
+ boolean negative = bytes[start] == '-';
+ if (negative || bytes[start] == '+') {
+ offset++;
+ if (length == 1) {
+ throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length));
+ }
+ }
+
+ return parse(bytes, start, length, offset, radix, negative);
+ }
+
+ private static int parse(byte[] bytes, int start, int length, int offset, int radix,
+ boolean negative) throws NumberFormatException {
+ int max = Integer.MIN_VALUE / radix;
+ int result = 0, end = start + length;
+ while (offset < end) {
+ int digit = LazyUtils.digit(bytes[offset++], radix);
+ if (digit == -1) {
+ throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length));
+ }
+ if (max > result) {
+ throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length));
+ }
+ int next = result * radix - digit;
+ if (next > result) {
+ throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length));
+ }
+ result = next;
+ }
+ if (!negative) {
+ result = -result;
+ if (result < 0) {
+ throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length));
+ }
+ }
+ return result;
+ }
+
+}
Added: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java?rev=745212&view=auto
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java (added)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java Tue Feb 17 19:39:29 2009
@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.lazy;
+
+/**
+ * LazyObject for storing a value of Long.
+ *
+ * <p>
+ * Part of the code is adapted from Apache Harmony Project.
+ *
+ * As with the specification, this implementation relied on code laid out in <a
+ * href="http://www.hackersdelight.org/">Henry S. Warren, Jr.'s Hacker's
+ * Delight, (Addison Wesley, 2002)</a> as well as <a
+ * href="http://aggregate.org/MAGIC/">The Aggregate's Magic Algorithms</a>.
+ * </p>
+ *
+ */
+public class LazyLong extends LazyPrimitive<Long> {
+
+ public LazyLong() {
+ super(Long.class);
+ }
+
+ @Override
+ public Long getPrimitiveObject() {
+ try {
+ // Slower method: convert to String and then convert to Long
+ // return Long.valueOf(LazyUtils.convertToString(bytes, start, length));
+ return Long.valueOf(parseLong(bytes, start, length));
+ } catch (NumberFormatException e) {
+ return null;
+ }
+ }
+
+ /**
+ * Parses the string argument as if it was a long value and returns the
+ * result. Throws NumberFormatException if the string does not represent a
+ * long quantity.
+ *
+ * @param bytes
+ * @param start
+ * @param length
+ * a UTF-8 encoded string representation of a long quantity.
+ * @return long the value represented by the argument
+ * @exception NumberFormatException
+ * if the argument could not be parsed as a long quantity.
+ */
+ public static long parseLong(byte[] bytes, int start, int length) throws NumberFormatException {
+ return parseLong(bytes, start, length, 10);
+ }
+
+ /**
+ * Parses the string argument as if it was an long value and returns the
+ * result. Throws NumberFormatException if the string does not represent an
+ * long quantity. The second argument specifies the radix to use when
+ * parsing the value.
+ *
+ * @param bytes
+ * @param start
+ * @param length
+ * a UTF-8 encoded string representation of a long quantity.
+ * @param radix
+ * the base to use for conversion.
+ * @return long the value represented by the argument
+ * @exception NumberFormatException
+ * if the argument could not be parsed as an long quantity.
+ */
+ public static long parseLong(byte[] bytes, int start, int length, int radix)
+ throws NumberFormatException {
+ if (bytes == null) {
+ throw new NumberFormatException("String is null");
+ }
+ if (radix < Character.MIN_RADIX ||
+ radix > Character.MAX_RADIX) {
+ throw new NumberFormatException("Invalid radix: " + radix);
+ }
+ if (length == 0) {
+ throw new NumberFormatException("Empty string!");
+ }
+ int offset = start;
+ boolean negative = bytes[start] == '-';
+ if (negative || bytes[start] == '+') {
+ offset ++;
+ if (length == 1) {
+ throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length));
+ }
+ }
+
+ return parse(bytes, start, length, offset, radix, negative);
+ }
+
+ private static long parse(byte[] bytes, int start, int length, int offset, int radix,
+ boolean negative) {
+ long max = Long.MIN_VALUE / radix;
+ long result = 0, end = start + length;
+ while (offset < end) {
+ int digit = LazyUtils.digit(bytes[offset++], radix);
+ if (digit == -1 || max > result) {
+ throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length));
+ }
+ long next = result * radix - digit;
+ if (next > result) {
+ throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length));
+ }
+ result = next;
+ }
+ if (!negative) {
+ result = -result;
+ if (result < 0) {
+ throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length));
+ }
+ }
+ return result;
+ }
+
+}
Added: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java?rev=745212&view=auto
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java (added)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java Tue Feb 17 19:39:29 2009
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.lazy;
+
+import org.apache.hadoop.io.Text;
+
+/**
+ * LazyObject stores an object in a range of bytes in a byte[].
+ *
+ * A LazyObject can represent anything.
+ *
+ */
+public class LazyObject {
+
+ protected byte[] bytes;
+ protected int start;
+ protected int length;
+
+ protected LazyObject() {
+ bytes = null;
+ start = 0;
+ length = 0;
+ }
+
+ protected LazyObject(byte[] bytes, int start, int length) {
+ setAll(bytes, start, length);
+ }
+
+ protected void setAll(byte[] bytes, int start, int length) {
+ this.bytes = bytes;
+ this.start = start;
+ this.length = length;
+ }
+
+}
Added: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java?rev=745212&view=auto
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java (added)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java Tue Feb 17 19:39:29 2009
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.lazy;
+
+
+/**
+ * LazyPrimitive stores a primitive Object in a LazyObject.
+ */
+public abstract class LazyPrimitive<T> extends LazyObject {
+
+ Class<T> primitiveClass;
+
+ protected LazyPrimitive(Class<T> primitiveClass) {
+ this.primitiveClass = primitiveClass;
+ }
+
+ /**
+ * Returns the actual primitive object represented by this LazyObject.
+ */
+ public abstract T getPrimitiveObject();
+
+}
Added: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyShort.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyShort.java?rev=745212&view=auto
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyShort.java (added)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyShort.java Tue Feb 17 19:39:29 2009
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.lazy;
+
+/**
+ * LazyObject for storing a value of Short.
+ *
+ * <p>
+ * Part of the code is adapted from Apache Harmony Project.
+ *
+ * As with the specification, this implementation relied on code laid out in <a
+ * href="http://www.hackersdelight.org/">Henry S. Warren, Jr.'s Hacker's
+ * Delight, (Addison Wesley, 2002)</a> as well as <a
+ * href="http://aggregate.org/MAGIC/">The Aggregate's Magic Algorithms</a>.
+ * </p>
+ *
+ */
+public class LazyShort extends LazyPrimitive<Short> {
+
+ public LazyShort() {
+ super(Short.class);
+ }
+
+ @Override
+ public Short getPrimitiveObject() {
+ try {
+ // Slower method: convert to String and then convert to Integer
+ // return Short.valueOf(LazyUtils.convertToString(bytes, start, length));
+ return Short.valueOf(parseShort(bytes, start, length));
+ } catch (NumberFormatException e) {
+ return null;
+ }
+ }
+
+ /**
+ * Parses the string argument as if it was a short value and returns the
+ * result. Throws NumberFormatException if the string does not represent an
+ * short quantity.
+ *
+ * @param bytes
+ * @param start
+ * @param length
+ * a UTF-8 encoded string representation of a short quantity.
+ * @return short the value represented by the argument
+ * @exception NumberFormatException
+ * if the argument could not be parsed as a short quantity.
+ */
+ public static short parseShort(byte[] bytes, int start, int length)
+ throws NumberFormatException {
+ return parseShort(bytes, start, length, 10);
+ }
+
+ /**
+ * Parses the string argument as if it was a short value and returns the
+ * result. Throws NumberFormatException if the string does not represent a
+ * single short quantity. The second argument specifies the radix to use
+ * when parsing the value.
+ *
+ * @param bytes
+ * @param start
+ * @param length
+ * a UTF-8 encoded string representation of a short quantity.
+ * @param radix
+ * the radix to use when parsing.
+ * @return short the value represented by the argument
+ * @exception NumberFormatException
+ * if the argument could not be parsed as a short quantity.
+ */
+ public static short parseShort(byte[] bytes, int start, int length, int radix)
+ throws NumberFormatException {
+ int intValue = LazyInteger.parseInt(bytes, start, length, radix);
+ short result = (short) intValue;
+ if (result == intValue) {
+ return result;
+ }
+ throw new NumberFormatException();
+ }
+
+}
Added: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java?rev=745212&view=auto
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java (added)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java Tue Feb 17 19:39:29 2009
@@ -0,0 +1,218 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.lazy;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.serde.Constants;
+import org.apache.hadoop.hive.serde2.SerDe;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.SerDeUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+
+/**
+ * LazySimpleSerDe can be used to read the same data format as
+ * MetadataTypedColumnsetSerDe and TCTLSeparatedProtocol.
+ *
+ * However, LazySimpleSerDe creates Objects in a lazy way, to
+ * provide better performance.
+ *
+ * Also LazySimpleSerDe outputs typed columns instead of treating
+ * all columns as String like MetadataTypedColumnsetSerDe.
+ */
+public class LazySimpleSerDe implements SerDe {
+
+ public static final Log LOG = LogFactory.getLog(LazySimpleSerDe.class.getName());
+
+ final public static byte DefaultSeparator = 1;
+ private byte separator;
+
+
+ private List<String> columnNames;
+ private List<String> columnTypes;
+ private ObjectInspector cachedObjectInspector;
+
+ private String nullString;
+ private boolean lastColumnTakesRest;
+
+ public String toString() {
+ return getClass().toString() + "[" + separator + ":"
+ + columnNames + ":" + columnTypes + "]";
+ }
+
+ public LazySimpleSerDe() throws SerDeException {
+ separator = DefaultSeparator;
+ }
+
+ private byte getByte(String altValue, byte defaultVal) {
+ if (altValue != null && altValue.length() > 0) {
+ try {
+ return Byte.valueOf(altValue).byteValue();
+ } catch(NumberFormatException e) {
+ return (byte)altValue.charAt(0);
+ }
+ }
+ return defaultVal;
+ }
+
+ /**
+ * Initialize the SerDe given the parameters.
+ * serialization.format: separator char or byte code (only supports byte-value up to 127)
+ * columns: ,-separated column naems
+ * columns.types: :-separated column types
+ */
+ public void initialize(Configuration job, Properties tbl) throws SerDeException {
+ // Read the separator
+ String alt_sep = tbl.getProperty(Constants.SERIALIZATION_FORMAT);
+ separator = getByte(alt_sep, DefaultSeparator);
+
+ // Read the configuration parameters
+ String columnNameProperty = tbl.getProperty("columns");
+ String columnTypeProperty = tbl.getProperty("columns.types");
+ if (columnNameProperty == null || columnTypeProperty == null) {
+ throw new SerDeException(getClass().toString()
+ + ": columns or columns.types property not set!");
+ }
+
+ nullString = tbl.getProperty(Constants.SERIALIZATION_NULL_FORMAT);
+ if (nullString == null) {
+ nullString = "\\N";
+ }
+
+ String lastColumnTakesRestString = tbl.getProperty(Constants.SERIALIZATION_LAST_COLUMN_TAKES_REST);
+ lastColumnTakesRest = (lastColumnTakesRestString != null && lastColumnTakesRestString.equalsIgnoreCase("true"));
+
+
+ // Parse the configuration parameters
+ columnNames = Arrays.asList(columnNameProperty.split(","));
+ columnTypes = Arrays.asList(columnTypeProperty.split(":"));
+ if (columnNames.size() != columnTypes.size()) {
+ throw new SerDeException(getClass().toString()
+ + ": columns has " + columnNames.size()
+ + " elements while columns.types has " + columnTypes.size() + " elements!");
+ }
+
+ // Create the LazyObject for storing the rows
+ LazyObject[] lazyPrimitives = new LazyObject[columnNames.size()];
+ // Create the ObjectInspectors for the fields
+ ArrayList<ObjectInspector> columnObjectInspectors
+ = new ArrayList<ObjectInspector>(columnNames.size());
+ for (int i=0; i<columnTypes.size(); i++) {
+ Class<?> primitiveClass = ObjectInspectorUtils.typeNameToClass.get( columnTypes.get(i) );
+ if (primitiveClass == null) {
+ throw new SerDeException(getClass().toString()
+ + ": type " + columnTypes.get(i) + " not supported!");
+ }
+ columnObjectInspectors.add(ObjectInspectorFactory.
+ getStandardPrimitiveObjectInspector(primitiveClass));
+ lazyPrimitives[i] = LazyUtils.createLazyPrimitiveClass(primitiveClass);
+ }
+
+ cachedObjectInspector =
+ ObjectInspectorFactory.getLazySimpleStructObjectInspector(columnNames,
+ columnObjectInspectors);
+
+ cachedLazyStruct = new LazyStruct(lazyPrimitives, separator,
+ new Text(nullString), lastColumnTakesRest);
+ }
+
+ // The object for storing row data
+ LazyStruct cachedLazyStruct;
+
+ /**
+ * Deserialize a row from the Writable to a LazyObject.
+ */
+ public Object deserialize(Writable field) throws SerDeException {
+ if (field instanceof BytesWritable) {
+ BytesWritable b = (BytesWritable)field;
+ // For backward-compatibility with hadoop 0.17
+ cachedLazyStruct.setAll(b.get(), 0, b.getSize());
+ } else if (field instanceof Text) {
+ Text t = (Text)field;
+ cachedLazyStruct.setAll(t.getBytes(), 0, t.getLength());
+ } else {
+ throw new SerDeException(getClass().toString()
+ + ": expects either BytesWritable or Text object!");
+ }
+ return cachedLazyStruct;
+ }
+
+
+ /**
+ * Returns the ObjectInspector for the row.
+ */
+ public ObjectInspector getObjectInspector() throws SerDeException {
+ return cachedObjectInspector;
+ }
+
+ /**
+ * Returns the Writable Class after serialization.
+ */
+ public Class<? extends Writable> getSerializedClass() {
+ return Text.class;
+ }
+
+ Text serializeCache = new Text();
+ /**
+ * Serialize a row of data.
+ */
+ public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException {
+
+ // TODO: We can switch the serialization to be directly based on
+ if (objInspector.getCategory() != Category.STRUCT) {
+ throw new SerDeException(getClass().toString()
+ + " can only serialize struct types, but we got: " + objInspector.getTypeName());
+ }
+ StructObjectInspector soi = (StructObjectInspector) objInspector;
+ List<? extends StructField> fields = soi.getAllStructFieldRefs();
+
+ StringBuilder sb = new StringBuilder();
+ for(int i=0; i<fields.size(); i++) {
+ if (i>0) {
+ sb.append((char)separator);
+ }
+ StructField field = fields.get(i);
+ Object fieldData = soi.getStructFieldData(obj, field);
+ if (field.getFieldObjectInspector().getCategory() == Category.PRIMITIVE) {
+ // For primitive object, serialize to plain string
+ sb.append(fieldData == null ? nullString : fieldData.toString());
+ } else {
+ // For complex object, serialize to JSON format
+ sb.append(SerDeUtils.getJSONString(fieldData, field.getFieldObjectInspector()));
+ }
+ }
+ serializeCache.set(sb.toString());
+ return serializeCache;
+ }
+
+}
Added: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java?rev=745212&view=auto
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java (added)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java Tue Feb 17 19:39:29 2009
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.lazy;
+
+import java.nio.charset.CharacterCodingException;
+
+import org.apache.hadoop.io.Text;
+
+/**
+ * LazyObject for storing a value of String.
+ */
+public class LazyString extends LazyPrimitive<String> {
+
+ public LazyString() {
+ super(String.class);
+ }
+
+ @Override
+ public String getPrimitiveObject() {
+ // In the future, we should allow returning a Text Object to save the UTF-8
+ // decoding/encoding, and the creation of new String object.
+ if (bytes == null) return null;
+ try {
+ return Text.decode(bytes, start, length);
+ } catch (CharacterCodingException e) {
+ return null;
+ }
+ }
+
+}
Added: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java?rev=745212&view=auto
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java (added)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java Tue Feb 17 19:39:29 2009
@@ -0,0 +1,159 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.lazy;
+
+import java.nio.charset.CharacterCodingException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.io.Text;
+
+
+/**
+ * LazyObject for storing a struct.
+ * The field of a struct can be primitive or non-primitive.
+ *
+ */
+public class LazyStruct extends LazyObject {
+
+
+ private static Log LOG = LogFactory.getLog(LazyStruct.class.getName());
+
+ LazyObject[] fields;
+ boolean[] fieldIsPrimitive;
+
+ byte separator;
+ Text nullSequence;
+ boolean lastColumnTakesAll;
+
+ boolean parsed;
+
+ /**
+ * Create a new LazyStruct Object.
+ * @param fields The field LazyObjects
+ * @param separator The separator for delimiting the fields in the byte[]
+ * @param nullSequence The sequence for null value
+ * @param lastColumnTakesAll whether the additional fields should be all put into the last column
+ * in case the data contains more columns than the schema.
+ */
+ public LazyStruct(LazyObject[] fields, byte separator,
+ Text nullSequence, boolean lastColumnTakesAll) {
+ this.fields = fields;
+ this.separator = separator;
+ this.nullSequence = nullSequence;
+ this.lastColumnTakesAll = lastColumnTakesAll;
+
+ parsed = false;
+ fieldIsPrimitive = new boolean[fields.length];
+ for(int i=0; i<fields.length; i++) {
+ fieldIsPrimitive[i] = (fields[i] instanceof LazyPrimitive);
+ }
+ }
+
+ /**
+ * Set the row data for this LazyStruct.
+ */
+ protected void setAll(byte[] bytes, int start, int length) {
+ super.setAll(bytes, start, length);
+ parsed = false;
+ }
+
+
+ boolean missingFieldWarned = false;
+ boolean extraFieldWarned = false;
+ /**
+ * Parse the byte[] and fill each field.
+ */
+ private void parse() {
+
+ int structByteEnd = start + length;
+ int fieldId = 0;
+ int fieldByteBegin = start;
+ int fieldByteEnd = start;
+
+ // Go through all bytes in the byte[]
+ while (fieldByteEnd <= structByteEnd) {
+ if (fieldByteEnd == structByteEnd || bytes[fieldByteEnd] == separator) {
+ // end of field reached
+ if (lastColumnTakesAll && fieldId == fields.length - 1) {
+ fieldByteEnd = structByteEnd;
+ }
+ // Test the length first so in most cases we avoid doing a byte[] comparison.
+ int fieldLength = fieldByteEnd - fieldByteBegin;
+ if (fieldLength == nullSequence.getLength()
+ && LazyUtils.compare(bytes, fieldByteBegin, fieldLength,
+ nullSequence.getBytes(), 0, nullSequence.getLength()) == 0) {
+ fields[fieldId].setAll(null, 0, 0);
+ } else {
+ fields[fieldId].setAll(bytes, fieldByteBegin,
+ fieldByteEnd - fieldByteBegin);
+ }
+ fieldId ++;
+ if (fieldId == fields.length || fieldByteEnd == structByteEnd) {
+ // all fields have been parsed, or all bytes have been parsed
+ break;
+ }
+ fieldByteBegin = fieldByteEnd + 1;
+ }
+ fieldByteEnd++;
+ }
+
+ // Extra bytes at the end?
+ if (!extraFieldWarned && fieldByteEnd < structByteEnd) {
+ extraFieldWarned = true;
+ LOG.warn("Extra bytes detected at the end of the row! Ignoring similar problems.");
+ }
+
+ // Missing fields?
+ if (!missingFieldWarned && fieldId < fields.length) {
+ missingFieldWarned = true;
+ LOG.warn("Missing fields! Expected " + fields.length + " fields but only got "
+ + fieldId + "! Ignoring similar problems.");
+ }
+
+ // Fill all missing fields with nulls.
+ for(; fieldId < fields.length; fieldId ++) {
+ fields[fieldId].setAll(null, 0, 0);
+ }
+
+ parsed = true;
+ }
+
+ /**
+ * Get one field out of the struct.
+ *
+ * If the field is a primitive field, return the actual object.
+ * Otherwise return the LazyObject. This is because PrimitiveObjectInspector
+ * does not have control over the object used by the user - the user simply
+ * directly use the Object instead of going through
+ * Object PrimitiveObjectInspector.get(Object).
+ *
+ * @param i the field ID
+ * @return the field as a LazyObject
+ */
+ public Object getField(int i) {
+ if (!parsed) {
+ parse();
+ }
+ if (!fieldIsPrimitive[i]) {
+ return fields[i];
+ } else {
+ return ((LazyPrimitive)fields[i]).getPrimitiveObject();
+ }
+ }
+}
Added: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java?rev=745212&view=auto
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java (added)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java Tue Feb 17 19:39:29 2009
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.lazy;
+
+import java.nio.charset.CharacterCodingException;
+
+import org.apache.hadoop.io.Text;
+
+public class LazyUtils {
+
+ /**
+ * Create a lazy primitive class given the java class.
+ */
+ public static LazyPrimitive<?> createLazyPrimitiveClass(Class<?> c) {
+ if (String.class.equals(c)) {
+ return new LazyString();
+ } else if (Integer.class.equals(c)) {
+ return new LazyInteger();
+ } else if (Double.class.equals(c)) {
+ return new LazyDouble();
+ } else if (Byte.class.equals(c)) {
+ return new LazyByte();
+ } else if (Short.class.equals(c)) {
+ return new LazyShort();
+ } else if (Long.class.equals(c)) {
+ return new LazyLong();
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Returns the digit represented by character b.
+ * @param b The ascii code of the character
+ * @param radix The radix
+ * @return -1 if it's invalid
+ */
+ public static int digit(int b, int radix) {
+ int r = -1;
+ if (b >= '0' && b<='9') {
+ r = b - '0';
+ } else if (b >= 'A' && b<='Z') {
+ r = b - 'A' + 10;
+ } else if (b >= 'a' && b <= 'z') {
+ r = b - 'a' + 10;
+ }
+ if (r >= radix) r = -1;
+ return r;
+ }
+
+ /**
+ * Returns -1 if the first byte sequence is lexicographically less than the second;
+ * returns +1 if the second byte sequence is lexicographically less than the first;
+ * otherwise return 0.
+ */
+ public static int compare(byte[] b1, int start1, int length1, byte[] b2, int start2, int length2) {
+
+ int min = Math.min(length1, length2);
+
+ for (int i = 0; i < min; i++) {
+ if (b1[start1 + i] == b2[start2 + i]) {
+ continue;
+ }
+ if (b1[start1 + i] < b2[start2 + i]) {
+ return -1;
+ } else {
+ return 1;
+ }
+ }
+
+ if (length1 < length2) return -1;
+ if (length1 > length2) return 1;
+ return 0;
+ }
+
+ /**
+ * Convert a UTF-8 byte array to String.
+ */
+ public static String convertToString(byte[] bytes, int start, int length) {
+ try {
+ return Text.decode(bytes, start, length);
+ } catch (CharacterCodingException e) {
+ return null;
+ }
+ }
+
+
+}
Added: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/LazySimpleStructObjectInspector.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/LazySimpleStructObjectInspector.java?rev=745212&view=auto
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/LazySimpleStructObjectInspector.java (added)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/LazySimpleStructObjectInspector.java Tue Feb 17 19:39:29 2009
@@ -0,0 +1,139 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.serde2.lazy.LazyStruct;
+
+/**
+ * LazySimpleStructObjectInspector works on struct data that is stored in LazyStruct.
+ * It only supports primitive types as its fields for simplicity and efficiency.
+ *
+ * The names of the struct fields and the internal structure of the struct fields
+ * are specified in the ctor of the LazySimpleStructObjectInspector.
+ *
+ * Always use the ObjectInspectorFactory to create new ObjectInspector objects, instead
+ * of directly creating an instance of this class.
+ */
+public class LazySimpleStructObjectInspector implements StructObjectInspector {
+
+ public static final Log LOG = LogFactory.getLog(LazySimpleStructObjectInspector.class.getName());
+
+ protected static class MyField implements StructField {
+ protected int fieldID;
+ protected String fieldName;
+ protected ObjectInspector fieldObjectInspector;
+
+ public MyField(int fieldID, String fieldName, ObjectInspector fieldObjectInspector) {
+ this.fieldID = fieldID;
+ this.fieldName = fieldName.toLowerCase();
+ this.fieldObjectInspector = fieldObjectInspector;
+ }
+
+ public int getFieldID() {
+ return fieldID;
+ }
+ public String getFieldName() {
+ return fieldName;
+ }
+ public ObjectInspector getFieldObjectInspector() {
+ return fieldObjectInspector;
+ }
+
+ public String toString() {
+ return "" + fieldID + ":" + fieldName;
+ }
+ }
+
+ protected List<MyField> fields;
+
+ public String getTypeName() {
+ return ObjectInspectorUtils.getStandardStructTypeName(this);
+ }
+
+ /** Call ObjectInspectorFactory.getLazySimpleStructObjectInspector instead.
+ */
+ protected LazySimpleStructObjectInspector(List<String> structFieldNames, List<ObjectInspector> structFieldObjectInspectors) {
+ init(structFieldNames, structFieldObjectInspectors);
+ }
+ protected void init(List<String> structFieldNames, List<ObjectInspector> structFieldObjectInspectors) {
+ assert(structFieldNames.size() == structFieldObjectInspectors.size());
+
+ fields = new ArrayList<MyField>(structFieldNames.size());
+ for(int i=0; i<structFieldNames.size(); i++) {
+ fields.add(new MyField(i, structFieldNames.get(i), structFieldObjectInspectors.get(i)));
+ }
+ }
+
+ protected LazySimpleStructObjectInspector(List<StructField> fields) {
+ init(fields);
+ }
+ protected void init(List<StructField> fields) {
+ this.fields = new ArrayList<MyField>(fields.size());
+ for(int i=0; i<fields.size(); i++) {
+ this.fields.add(new MyField(i, fields.get(i).getFieldName(), fields.get(i).getFieldObjectInspector()));
+ }
+ }
+
+
+ public final Category getCategory() {
+ return Category.STRUCT;
+ }
+
+ // Without Data
+ public StructField getStructFieldRef(String fieldName) {
+ return ObjectInspectorUtils.getStandardStructFieldRef(fieldName, fields);
+ }
+ public List<? extends StructField> getAllStructFieldRefs() {
+ return fields;
+ }
+
+ // With Data
+ @SuppressWarnings("unchecked")
+ public Object getStructFieldData(Object data, StructField fieldRef) {
+ if (data == null) {
+ return null;
+ }
+ LazyStruct struct = (LazyStruct)data;
+ MyField f = (MyField) fieldRef;
+
+ int fieldID = f.getFieldID();
+ assert(fieldID >= 0 && fieldID < fields.size());
+
+ return struct.getField(fieldID);
+ }
+
+ @Override
+ public List<Object> getStructFieldsDataAsList(Object data) {
+ if (data == null) {
+ return null;
+ }
+ List<Object> fieldsData = new ArrayList<Object>(fields.size());
+ LazyStruct struct = (LazyStruct)data;
+ for (int i=0; i<fields.size(); i++) {
+ fieldsData.add(struct.getField(i));
+ }
+ return fieldsData;
+ }
+
+}
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java?rev=745212&r1=745211&r2=745212&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java Tue Feb 17 19:39:29 2009
@@ -180,6 +180,20 @@
}
return result;
}
+
+ static HashMap<ArrayList<List<?>>, LazySimpleStructObjectInspector> cachedLazySimpleStructObjectInspector =
+ new HashMap<ArrayList<List<?>>, LazySimpleStructObjectInspector>();
+ public static LazySimpleStructObjectInspector getLazySimpleStructObjectInspector(List<String> structFieldNames, List<ObjectInspector> structFieldObjectInspectors) {
+ ArrayList<List<?>> signature = new ArrayList<List<?>>();
+ signature.add(structFieldNames);
+ signature.add(structFieldObjectInspectors);
+ LazySimpleStructObjectInspector result = cachedLazySimpleStructObjectInspector.get(signature);
+ if (result == null) {
+ result = new LazySimpleStructObjectInspector(structFieldNames, structFieldObjectInspectors);
+ cachedLazySimpleStructObjectInspector.put(signature, result);
+ }
+ return result;
+ }
static HashMap<List<StructObjectInspector>, UnionStructObjectInspector> cachedUnionStructObjectInspector =
new HashMap<List<StructObjectInspector>, UnionStructObjectInspector>();
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java?rev=745212&r1=745211&r2=745212&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java Tue Feb 17 19:39:29 2009
@@ -25,6 +25,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.serde.Constants;
/**
* ObjectInspectorFactory is the primary way to create new ObjectInspector instances.
@@ -75,6 +76,26 @@
classToTypeName.put(String.class, org.apache.hadoop.hive.serde.Constants.STRING_TYPE_NAME);
classToTypeName.put(java.sql.Date.class, org.apache.hadoop.hive.serde.Constants.DATE_TYPE_NAME);
}
+
+ /**
+ * The mapping from type name in DDL to the Java class.
+ */
+ public static final Map<String, Class<?>> typeNameToClass = new HashMap<String, Class<?>>();
+ static {
+ typeNameToClass.put(Constants.BOOLEAN_TYPE_NAME, Boolean.class);
+ typeNameToClass.put(Constants.TINYINT_TYPE_NAME, Byte.class);
+ typeNameToClass.put(Constants.SMALLINT_TYPE_NAME, Short.class);
+ typeNameToClass.put(Constants.INT_TYPE_NAME, Integer.class);
+ typeNameToClass.put(Constants.BIGINT_TYPE_NAME, Long.class);
+ typeNameToClass.put(Constants.FLOAT_TYPE_NAME, Float.class);
+ typeNameToClass.put(Constants.DOUBLE_TYPE_NAME, Double.class);
+ typeNameToClass.put(Constants.STRING_TYPE_NAME, String.class);
+ typeNameToClass.put(Constants.DATE_TYPE_NAME, java.sql.Date.class);
+ // These types are not supported yet.
+ // TypeNameToClass.put(Constants.DATETIME_TYPE_NAME);
+ // TypeNameToClass.put(Constants.TIMESTAMP_TYPE_NAME);
+ }
+
/**
* Get the short name for the types
*/