You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by zs...@apache.org on 2009/02/24 08:18:50 UTC
svn commit: r747293 [6/6] - in /hadoop/hive/trunk: ./
metastore/src/java/org/apache/hadoop/hive/metastore/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/metadata/
ql/src/java/org/apache/hadoop/hive/ql/parse/ ql/src/j...
Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/udf6.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/udf6.q.xml?rev=747293&r1=747292&r2=747293&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/udf6.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/udf6.q.xml Tue Feb 24 07:18:47 2009
@@ -26,12 +26,12 @@
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc">
<void property="dirName">
- <string>/data/users/zshao/sync/apache-trunk-HIVE-270/ql/../build/ql/tmp/343375627.10001.insclause-0</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/602355685.10001.insclause-0</string>
</void>
<void property="tableInfo">
<object class="org.apache.hadoop.hive.ql.plan.tableDesc">
<void property="deserializerClass">
- <class>org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe</class>
+ <class>org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe</class>
</void>
<void property="inputFileFormatClass">
<class>org.apache.hadoop.mapred.TextInputFormat</class>
@@ -295,7 +295,7 @@
<void property="pathToAliases">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src</string>
<object class="java.util.ArrayList">
<void method="add">
<string>src</string>
@@ -307,7 +307,7 @@
<void property="pathToPartitionInfo">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src</string>
<object class="org.apache.hadoop.hive.ql.plan.partitionDesc">
<void property="partSpec">
<object class="java.util.LinkedHashMap"/>
@@ -315,7 +315,7 @@
<void property="tableDesc">
<object class="org.apache.hadoop.hive.ql.plan.tableDesc">
<void property="deserializerClass">
- <class>org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe</class>
+ <class>org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe</class>
</void>
<void property="inputFileFormatClass">
<class>org.apache.hadoop.mapred.TextInputFormat</class>
@@ -351,7 +351,7 @@
</void>
<void method="put">
<string>serialization.lib</string>
- <string>org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe</string>
+ <string>org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe</string>
</void>
<void method="put">
<string>file.inputformat</string>
@@ -363,12 +363,12 @@
</void>
<void method="put">
<string>location</string>
- <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src</string>
</void>
</object>
</void>
<void property="serdeClassName">
- <string>org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe</string>
+ <string>org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe</string>
</void>
</object>
</void>
Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/union.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/union.q.xml?rev=747293&r1=747292&r2=747293&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/union.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/union.q.xml Tue Feb 24 07:18:47 2009
@@ -28,7 +28,7 @@
<boolean>true</boolean>
</void>
<void property="sourceDir">
- <string>/data/users/zshao/sync/apache-trunk-HIVE-270/ql/../build/ql/tmp/449888396/1582541203.10000.insclause-0</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/83128511/25205679.10000.insclause-0</string>
</void>
<void property="targetDir">
<string>../build/ql/test/data/warehouse/union.out</string>
@@ -79,12 +79,12 @@
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc">
<void property="dirName">
- <string>/data/users/zshao/sync/apache-trunk-HIVE-270/ql/../build/ql/tmp/449888396/1582541203.10000.insclause-0</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-292/ql/../build/ql/tmp/83128511/25205679.10000.insclause-0</string>
</void>
<void property="tableInfo">
<object class="org.apache.hadoop.hive.ql.plan.tableDesc">
<void property="deserializerClass">
- <class>org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe</class>
+ <class>org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe</class>
</void>
<void property="inputFileFormatClass">
<class>org.apache.hadoop.mapred.TextInputFormat</class>
@@ -690,7 +690,7 @@
<void property="pathToAliases">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src</string>
<object class="java.util.ArrayList">
<void method="add">
<string>null-subquery1:unioninput-subquery1:src</string>
@@ -705,7 +705,7 @@
<void property="pathToPartitionInfo">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src</string>
<object class="org.apache.hadoop.hive.ql.plan.partitionDesc">
<void property="partSpec">
<object class="java.util.LinkedHashMap"/>
@@ -713,7 +713,7 @@
<void property="tableDesc">
<object class="org.apache.hadoop.hive.ql.plan.tableDesc">
<void property="deserializerClass">
- <class>org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe</class>
+ <class>org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe</class>
</void>
<void property="inputFileFormatClass">
<class>org.apache.hadoop.mapred.TextInputFormat</class>
@@ -749,7 +749,7 @@
</void>
<void method="put">
<string>serialization.lib</string>
- <string>org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe</string>
+ <string>org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe</string>
</void>
<void method="put">
<string>file.inputformat</string>
@@ -761,12 +761,12 @@
</void>
<void method="put">
<string>location</string>
- <string>file:/data/users/zshao/sync/apache-trunk-HIVE-270/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-292/build/ql/test/data/warehouse/src</string>
</void>
</object>
</void>
<void property="serdeClassName">
- <string>org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe</string>
+ <string>org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe</string>
</void>
</object>
</void>
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java?rev=747293&r1=747292&r2=747293&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java Tue Feb 24 07:18:47 2009
@@ -84,6 +84,7 @@
try {
// loading these classes will automatically register the short names
Class.forName(org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe.class.getName());
+ Class.forName(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName());
Class.forName(org.apache.hadoop.hive.serde2.ThriftDeserializer.class.getName());
} catch (ClassNotFoundException e) {
throw new RuntimeException("IMPOSSIBLE Exception: Unable to initialize core serdes", e);
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java?rev=747293&r1=747292&r2=747293&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java Tue Feb 24 07:18:47 2009
@@ -27,6 +27,7 @@
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.serde.Constants;
+import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe;
import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.SerDeUtils;
@@ -98,11 +99,8 @@
// Read the configuration parameters
String columnNameProperty = tbl.getProperty("columns");
+ // NOTE: if "columns.types" is missing, all columns will be of String type
String columnTypeProperty = tbl.getProperty("columns.types");
- if (columnNameProperty == null || columnTypeProperty == null) {
- throw new SerDeException(getClass().toString()
- + ": columns or columns.types property not set!");
- }
nullString = tbl.getProperty(Constants.SERIALIZATION_NULL_FORMAT);
if (nullString == null) {
@@ -111,11 +109,22 @@
String lastColumnTakesRestString = tbl.getProperty(Constants.SERIALIZATION_LAST_COLUMN_TAKES_REST);
lastColumnTakesRest = (lastColumnTakesRestString != null && lastColumnTakesRestString.equalsIgnoreCase("true"));
-
// Parse the configuration parameters
- columnNames = Arrays.asList(columnNameProperty.split(","));
- columnTypes = Arrays.asList(columnTypeProperty.split(":"));
+ if (columnNameProperty != null) {
+ columnNames = Arrays.asList(columnNameProperty.split(","));
+ } else {
+ columnNames = new ArrayList<String>();
+ }
+ if (columnTypeProperty != null) {
+ columnTypes = Arrays.asList(columnTypeProperty.split(":"));
+ } else {
+ // Default type: all string
+ columnTypes = new ArrayList<String>();
+ for (int i = 0; i < columnNames.size(); i++) {
+ columnTypes.add(Constants.STRING_TYPE_NAME);
+ }
+ }
if (columnNames.size() != columnTypes.size()) {
throw new SerDeException(getClass().toString()
+ ": columns has " + columnNames.size()
@@ -144,6 +153,10 @@
cachedLazyStruct = new LazyStruct(lazyPrimitives, separator,
new Text(nullString), lastColumnTakesRest);
+
+ LOG.debug("LazySimpleSerDe initialized with: columnNames=" + columnNames + " columnTypes="
+ + columnTypes + " separator=" + separator + " nullstring=" + nullString
+ + " lastColumnTakesRest=" + lastColumnTakesRest);
}
// The object for storing row data