svn commit: r712905 [33/38] - in /hadoop/core/trunk: ./ src/contrib/hive/ src/contrib/hive/cli/src/java/org/apache/hadoop/hive/cli/ src/contrib/hive/common/src/java/org/apache/hadoop/hive/conf/ src/contrib/hive/conf/ src/contrib/hive/data/files/ src/co...

[dh...@apache.org]

Modified: hadoop/core/trunk/src/contrib/hive/ql/src/test/results/compiler/plan/union.q.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/test/results/compiler/plan/union.q.xml?rev=712905&r1=712904&r2=712905&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/ql/src/test/results/compiler/plan/union.q.xml (original)
+++ hadoop/core/trunk/src/contrib/hive/ql/src/test/results/compiler/plan/union.q.xml Mon Nov 10 17:50:06 2008
@@ -28,7 +28,7 @@
              <boolean>true</boolean> 
             </void> 
             <void property="sourceDir"> 
-             <string>/tmp/hive-njain/635791418/469963157.10000.insclause-0</string> 
+             <string>/tmp/hive-njain/14629339/1349764315.10000.insclause-0</string> 
             </void> 
             <void property="targetDir"> 
              <string>../../../../build/contrib/hive/ql/test/data/warehouse/union.out</string> 
@@ -55,23 +55,23 @@
      <object class="java.util.HashMap"> 
       <void method="put"> 
        <string>null-subquery1:unioninput-subquery1:src</string> 
-       <object class="org.apache.hadoop.hive.ql.exec.ForwardOperator"> 
+       <object id="ForwardOperator0" class="org.apache.hadoop.hive.ql.exec.ForwardOperator"> 
         <void property="childOperators"> 
          <object class="java.util.ArrayList"> 
           <void method="add"> 
-           <object class="org.apache.hadoop.hive.ql.exec.FilterOperator"> 
+           <object id="FilterOperator0" class="org.apache.hadoop.hive.ql.exec.FilterOperator"> 
             <void property="childOperators"> 
              <object class="java.util.ArrayList"> 
               <void method="add"> 
-               <object class="org.apache.hadoop.hive.ql.exec.SelectOperator"> 
+               <object id="SelectOperator0" class="org.apache.hadoop.hive.ql.exec.SelectOperator"> 
                 <void property="childOperators"> 
                  <object id="ArrayList0" class="java.util.ArrayList"> 
                   <void method="add"> 
-                   <object class="org.apache.hadoop.hive.ql.exec.ForwardOperator"> 
+                   <object id="ForwardOperator1" class="org.apache.hadoop.hive.ql.exec.ForwardOperator"> 
                     <void property="childOperators"> 
                      <object class="java.util.ArrayList"> 
                       <void method="add"> 
-                       <object class="org.apache.hadoop.hive.ql.exec.SelectOperator"> 
+                       <object id="SelectOperator1" class="org.apache.hadoop.hive.ql.exec.SelectOperator"> 
                         <void property="childOperators"> 
                          <object class="java.util.ArrayList"> 
                           <void method="add"> 
@@ -79,7 +79,7 @@
                             <void property="conf"> 
                              <object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc"> 
                               <void property="dirName"> 
-                               <string>/tmp/hive-njain/635791418/469963157.10000.insclause-0</string> 
+                               <string>/tmp/hive-njain/14629339/1349764315.10000.insclause-0</string> 
                               </void> 
                               <void property="tableInfo"> 
                                <object class="org.apache.hadoop.hive.ql.plan.tableDesc"> 
@@ -95,6 +95,10 @@
                                 <void property="properties"> 
                                  <object class="java.util.Properties"> 
                                   <void method="put"> 
+                                   <string>columns</string> 
+                                   <string>key,value</string> 
+                                  </void> 
+                                  <void method="put"> 
                                    <string>serialization.format</string> 
                                    <string>1</string> 
                                   </void> 
@@ -104,6 +108,13 @@
                               </void> 
                              </object> 
                             </void> 
+                            <void property="parentOperators"> 
+                             <object class="java.util.ArrayList"> 
+                              <void method="add"> 
+                               <object idref="SelectOperator1"/> 
+                              </void> 
+                             </object> 
+                            </void> 
                             <void property="schema"> 
                              <object class="org.apache.hadoop.hive.ql.exec.RowSchema"> 
                               <void property="signature"> 
@@ -166,6 +177,16 @@
                             </void> 
                            </object> 
                           </void> 
+                          <void property="selectStar"> 
+                           <boolean>true</boolean> 
+                          </void> 
+                         </object> 
+                        </void> 
+                        <void property="parentOperators"> 
+                         <object class="java.util.ArrayList"> 
+                          <void method="add"> 
+                           <object idref="ForwardOperator1"/> 
+                          </void> 
                          </object> 
                         </void> 
                         <void property="schema"> 
@@ -179,6 +200,203 @@
                       </void> 
                      </object> 
                     </void> 
+                    <void property="parentOperators"> 
+                     <object class="java.util.ArrayList"> 
+                      <void method="add"> 
+                       <object idref="SelectOperator0"/> 
+                      </void> 
+                      <void method="add"> 
+                       <object id="SelectOperator2" class="org.apache.hadoop.hive.ql.exec.SelectOperator"> 
+                        <void property="childOperators"> 
+                         <object idref="ArrayList0"/> 
+                        </void> 
+                        <void property="conf"> 
+                         <object class="org.apache.hadoop.hive.ql.plan.selectDesc"> 
+                          <void property="colList"> 
+                           <object class="java.util.ArrayList"> 
+                            <void method="add"> 
+                             <object class="org.apache.hadoop.hive.ql.plan.exprNodeColumnDesc"> 
+                              <void property="column"> 
+                               <string>key</string> 
+                              </void> 
+                              <void property="typeInfo"> 
+                               <object idref="PrimitiveTypeInfo0"/> 
+                              </void> 
+                             </object> 
+                            </void> 
+                            <void method="add"> 
+                             <object class="org.apache.hadoop.hive.ql.plan.exprNodeColumnDesc"> 
+                              <void property="column"> 
+                               <string>value</string> 
+                              </void> 
+                              <void property="typeInfo"> 
+                               <object idref="PrimitiveTypeInfo0"/> 
+                              </void> 
+                             </object> 
+                            </void> 
+                           </object> 
+                          </void> 
+                          <void property="selectStar"> 
+                           <boolean>true</boolean> 
+                          </void> 
+                         </object> 
+                        </void> 
+                        <void property="parentOperators"> 
+                         <object class="java.util.ArrayList"> 
+                          <void method="add"> 
+                           <object id="FilterOperator1" class="org.apache.hadoop.hive.ql.exec.FilterOperator"> 
+                            <void property="childOperators"> 
+                             <object class="java.util.ArrayList"> 
+                              <void method="add"> 
+                               <object idref="SelectOperator2"/> 
+                              </void> 
+                             </object> 
+                            </void> 
+                            <void property="conf"> 
+                             <object class="org.apache.hadoop.hive.ql.plan.filterDesc"> 
+                              <void property="predicate"> 
+                               <object class="org.apache.hadoop.hive.ql.plan.exprNodeFuncDesc"> 
+                                <void property="UDFClass"> 
+                                 <class>org.apache.hadoop.hive.ql.udf.UDFOPGreaterThan</class> 
+                                </void> 
+                                <void property="UDFMethod"> 
+                                 <object class="org.apache.hadoop.hive.ql.udf.UDFBaseCompare" method="getMethod"> 
+                                  <string>evaluate</string> 
+                                  <array class="java.lang.Class" length="2"> 
+                                   <void index="0"> 
+                                    <class>java.lang.String</class> 
+                                   </void> 
+                                   <void index="1"> 
+                                    <class>java.lang.Number</class> 
+                                   </void> 
+                                  </array> 
+                                 </object> 
+                                </void> 
+                                <void property="children"> 
+                                 <object class="java.util.ArrayList"> 
+                                  <void method="add"> 
+                                   <object class="org.apache.hadoop.hive.ql.plan.exprNodeColumnDesc"> 
+                                    <void property="column"> 
+                                     <string>key</string> 
+                                    </void> 
+                                    <void property="typeInfo"> 
+                                     <object idref="PrimitiveTypeInfo0"/> 
+                                    </void> 
+                                   </object> 
+                                  </void> 
+                                  <void method="add"> 
+                                   <object class="org.apache.hadoop.hive.ql.plan.exprNodeConstantDesc"> 
+                                    <void property="typeInfo"> 
+                                     <object id="PrimitiveTypeInfo1" class="org.apache.hadoop.hive.ql.typeinfo.PrimitiveTypeInfo"> 
+                                      <void property="primitiveClass"> 
+                                       <class>java.lang.Integer</class> 
+                                      </void> 
+                                     </object> 
+                                    </void> 
+                                    <void property="value"> 
+                                     <int>100</int> 
+                                    </void> 
+                                   </object> 
+                                  </void> 
+                                 </object> 
+                                </void> 
+                                <void property="typeInfo"> 
+                                 <object id="PrimitiveTypeInfo2" class="org.apache.hadoop.hive.ql.typeinfo.PrimitiveTypeInfo"> 
+                                  <void property="primitiveClass"> 
+                                   <class>java.lang.Boolean</class> 
+                                  </void> 
+                                 </object> 
+                                </void> 
+                               </object> 
+                              </void> 
+                             </object> 
+                            </void> 
+                            <void property="parentOperators"> 
+                             <object class="java.util.ArrayList"> 
+                              <void method="add"> 
+                               <object id="ForwardOperator2" class="org.apache.hadoop.hive.ql.exec.ForwardOperator"> 
+                                <void property="childOperators"> 
+                                 <object class="java.util.ArrayList"> 
+                                  <void method="add"> 
+                                   <object idref="FilterOperator1"/> 
+                                  </void> 
+                                 </object> 
+                                </void> 
+                                <void property="schema"> 
+                                 <object class="org.apache.hadoop.hive.ql.exec.RowSchema"> 
+                                  <void property="signature"> 
+                                   <object id="Vector1" class="java.util.Vector"> 
+                                    <void method="add"> 
+                                     <object class="org.apache.hadoop.hive.ql.exec.ColumnInfo"> 
+                                      <void property="internalName"> 
+                                       <string>key</string> 
+                                      </void> 
+                                      <void property="type"> 
+                                       <object idref="PrimitiveTypeInfo0"/> 
+                                      </void> 
+                                     </object> 
+                                    </void> 
+                                    <void method="add"> 
+                                     <object class="org.apache.hadoop.hive.ql.exec.ColumnInfo"> 
+                                      <void property="internalName"> 
+                                       <string>value</string> 
+                                      </void> 
+                                      <void property="type"> 
+                                       <object idref="PrimitiveTypeInfo0"/> 
+                                      </void> 
+                                     </object> 
+                                    </void> 
+                                   </object> 
+                                  </void> 
+                                 </object> 
+                                </void> 
+                               </object> 
+                              </void> 
+                             </object> 
+                            </void> 
+                            <void property="schema"> 
+                             <object class="org.apache.hadoop.hive.ql.exec.RowSchema"> 
+                              <void property="signature"> 
+                               <object idref="Vector1"/> 
+                              </void> 
+                             </object> 
+                            </void> 
+                           </object> 
+                          </void> 
+                         </object> 
+                        </void> 
+                        <void property="schema"> 
+                         <object class="org.apache.hadoop.hive.ql.exec.RowSchema"> 
+                          <void property="signature"> 
+                           <object class="java.util.Vector"> 
+                            <void method="add"> 
+                             <object class="org.apache.hadoop.hive.ql.exec.ColumnInfo"> 
+                              <void property="internalName"> 
+                               <string>0</string> 
+                              </void> 
+                              <void property="type"> 
+                               <object idref="PrimitiveTypeInfo0"/> 
+                              </void> 
+                             </object> 
+                            </void> 
+                            <void method="add"> 
+                             <object class="org.apache.hadoop.hive.ql.exec.ColumnInfo"> 
+                              <void property="internalName"> 
+                               <string>1</string> 
+                              </void> 
+                              <void property="type"> 
+                               <object idref="PrimitiveTypeInfo0"/> 
+                              </void> 
+                             </object> 
+                            </void> 
+                           </object> 
+                          </void> 
+                         </object> 
+                        </void> 
+                       </object> 
+                      </void> 
+                     </object> 
+                    </void> 
                     <void property="schema"> 
                      <object class="org.apache.hadoop.hive.ql.exec.RowSchema"> 
                       <void property="signature"> 
@@ -239,6 +457,13 @@
                   </void> 
                  </object> 
                 </void> 
+                <void property="parentOperators"> 
+                 <object class="java.util.ArrayList"> 
+                  <void method="add"> 
+                   <object idref="FilterOperator0"/> 
+                  </void> 
+                 </object> 
+                </void> 
                 <void property="schema"> 
                  <object class="org.apache.hadoop.hive.ql.exec.RowSchema"> 
                   <void property="signature"> 
@@ -292,11 +517,7 @@
                   <void method="add"> 
                    <object class="org.apache.hadoop.hive.ql.plan.exprNodeConstantDesc"> 
                     <void property="typeInfo"> 
-                     <object id="PrimitiveTypeInfo1" class="org.apache.hadoop.hive.ql.typeinfo.PrimitiveTypeInfo"> 
-                      <void property="primitiveClass"> 
-                       <class>java.lang.Integer</class> 
-                      </void> 
-                     </object> 
+                     <object idref="PrimitiveTypeInfo1"/> 
                     </void> 
                     <void property="value"> 
                      <int>100</int> 
@@ -306,179 +527,16 @@
                  </object> 
                 </void> 
                 <void property="typeInfo"> 
-                 <object id="PrimitiveTypeInfo2" class="org.apache.hadoop.hive.ql.typeinfo.PrimitiveTypeInfo"> 
-                  <void property="primitiveClass"> 
-                   <class>java.lang.Boolean</class> 
-                  </void> 
-                 </object> 
-                </void> 
-               </object> 
-              </void> 
-             </object> 
-            </void> 
-            <void property="schema"> 
-             <object class="org.apache.hadoop.hive.ql.exec.RowSchema"> 
-              <void property="signature"> 
-               <object id="Vector1" class="java.util.Vector"> 
-                <void method="add"> 
-                 <object class="org.apache.hadoop.hive.ql.exec.ColumnInfo"> 
-                  <void property="internalName"> 
-                   <string>key</string> 
-                  </void> 
-                  <void property="type"> 
-                   <object idref="PrimitiveTypeInfo0"/> 
-                  </void> 
-                 </object> 
-                </void> 
-                <void method="add"> 
-                 <object class="org.apache.hadoop.hive.ql.exec.ColumnInfo"> 
-                  <void property="internalName"> 
-                   <string>value</string> 
-                  </void> 
-                  <void property="type"> 
-                   <object idref="PrimitiveTypeInfo0"/> 
-                  </void> 
-                 </object> 
+                 <object idref="PrimitiveTypeInfo2"/> 
                 </void> 
                </object> 
               </void> 
              </object> 
             </void> 
-           </object> 
-          </void> 
-         </object> 
-        </void> 
-        <void property="schema"> 
-         <object class="org.apache.hadoop.hive.ql.exec.RowSchema"> 
-          <void property="signature"> 
-           <object idref="Vector1"/> 
-          </void> 
-         </object> 
-        </void> 
-       </object> 
-      </void> 
-      <void method="put"> 
-       <string>null-subquery2:unioninput-subquery2:src</string> 
-       <object class="org.apache.hadoop.hive.ql.exec.ForwardOperator"> 
-        <void property="childOperators"> 
-         <object class="java.util.ArrayList"> 
-          <void method="add"> 
-           <object class="org.apache.hadoop.hive.ql.exec.FilterOperator"> 
-            <void property="childOperators"> 
+            <void property="parentOperators"> 
              <object class="java.util.ArrayList"> 
               <void method="add"> 
-               <object class="org.apache.hadoop.hive.ql.exec.SelectOperator"> 
-                <void property="childOperators"> 
-                 <object idref="ArrayList0"/> 
-                </void> 
-                <void property="conf"> 
-                 <object class="org.apache.hadoop.hive.ql.plan.selectDesc"> 
-                  <void property="colList"> 
-                   <object class="java.util.ArrayList"> 
-                    <void method="add"> 
-                     <object class="org.apache.hadoop.hive.ql.plan.exprNodeColumnDesc"> 
-                      <void property="column"> 
-                       <string>key</string> 
-                      </void> 
-                      <void property="typeInfo"> 
-                       <object idref="PrimitiveTypeInfo0"/> 
-                      </void> 
-                     </object> 
-                    </void> 
-                    <void method="add"> 
-                     <object class="org.apache.hadoop.hive.ql.plan.exprNodeColumnDesc"> 
-                      <void property="column"> 
-                       <string>value</string> 
-                      </void> 
-                      <void property="typeInfo"> 
-                       <object idref="PrimitiveTypeInfo0"/> 
-                      </void> 
-                     </object> 
-                    </void> 
-                   </object> 
-                  </void> 
-                 </object> 
-                </void> 
-                <void property="schema"> 
-                 <object class="org.apache.hadoop.hive.ql.exec.RowSchema"> 
-                  <void property="signature"> 
-                   <object class="java.util.Vector"> 
-                    <void method="add"> 
-                     <object class="org.apache.hadoop.hive.ql.exec.ColumnInfo"> 
-                      <void property="internalName"> 
-                       <string>0</string> 
-                      </void> 
-                      <void property="type"> 
-                       <object idref="PrimitiveTypeInfo0"/> 
-                      </void> 
-                     </object> 
-                    </void> 
-                    <void method="add"> 
-                     <object class="org.apache.hadoop.hive.ql.exec.ColumnInfo"> 
-                      <void property="internalName"> 
-                       <string>1</string> 
-                      </void> 
-                      <void property="type"> 
-                       <object idref="PrimitiveTypeInfo0"/> 
-                      </void> 
-                     </object> 
-                    </void> 
-                   </object> 
-                  </void> 
-                 </object> 
-                </void> 
-               </object> 
-              </void> 
-             </object> 
-            </void> 
-            <void property="conf"> 
-             <object class="org.apache.hadoop.hive.ql.plan.filterDesc"> 
-              <void property="predicate"> 
-               <object class="org.apache.hadoop.hive.ql.plan.exprNodeFuncDesc"> 
-                <void property="UDFClass"> 
-                 <class>org.apache.hadoop.hive.ql.udf.UDFOPGreaterThan</class> 
-                </void> 
-                <void property="UDFMethod"> 
-                 <object class="org.apache.hadoop.hive.ql.udf.UDFBaseCompare" method="getMethod"> 
-                  <string>evaluate</string> 
-                  <array class="java.lang.Class" length="2"> 
-                   <void index="0"> 
-                    <class>java.lang.String</class> 
-                   </void> 
-                   <void index="1"> 
-                    <class>java.lang.Number</class> 
-                   </void> 
-                  </array> 
-                 </object> 
-                </void> 
-                <void property="children"> 
-                 <object class="java.util.ArrayList"> 
-                  <void method="add"> 
-                   <object class="org.apache.hadoop.hive.ql.plan.exprNodeColumnDesc"> 
-                    <void property="column"> 
-                     <string>key</string> 
-                    </void> 
-                    <void property="typeInfo"> 
-                     <object idref="PrimitiveTypeInfo0"/> 
-                    </void> 
-                   </object> 
-                  </void> 
-                  <void method="add"> 
-                   <object class="org.apache.hadoop.hive.ql.plan.exprNodeConstantDesc"> 
-                    <void property="typeInfo"> 
-                     <object idref="PrimitiveTypeInfo1"/> 
-                    </void> 
-                    <void property="value"> 
-                     <int>100</int> 
-                    </void> 
-                   </object> 
-                  </void> 
-                 </object> 
-                </void> 
-                <void property="typeInfo"> 
-                 <object idref="PrimitiveTypeInfo2"/> 
-                </void> 
-               </object> 
+               <object idref="ForwardOperator0"/> 
               </void> 
              </object> 
             </void> 
@@ -523,6 +581,10 @@
         </void> 
        </object> 
       </void> 
+      <void method="put"> 
+       <string>null-subquery2:unioninput-subquery2:src</string> 
+       <object idref="ForwardOperator2"/> 
+      </void> 
      </object> 
     </void> 
     <void property="pathToAliases"> 
@@ -608,6 +670,9 @@
       </void> 
      </object> 
     </void> 
+    <void property="tagToValueDesc"> 
+     <object class="java.util.ArrayList"/> 
+    </void> 
    </object> 
   </void> 
  </object> 

Added: hadoop/core/trunk/src/contrib/hive/ql/src/test/scripts/testgrep
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/ql/src/test/scripts/testgrep?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/ql/src/test/scripts/testgrep (added)
+++ hadoop/core/trunk/src/contrib/hive/ql/src/test/scripts/testgrep Mon Nov 10 17:50:06 2008
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+egrep '10.*'
+
+exit 0;

Added: hadoop/core/trunk/src/contrib/hive/serde/README
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/README?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/README (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/README Mon Nov 10 17:50:06 2008
@@ -0,0 +1,115 @@
+What is SerDe
+-----------
+SerDe is a short name for Serializer and Deserializer.
+Hive uses SerDe (and FileFormat) to read from/write to tables.
+
+* HDFS files --(InputFileFormat)--> <key, value> --(Deserializer)--> Row object
+* Row object --(Serializer)--> <key, value> --(OutputFileFormat)--> HDFS files
+
+Note that the "key" part is ignored when reading, and is always a constant when
+writing. Basically the row object is only stored into the "value".
+
+One principle of Hive is that Hive does not own the HDFS file format - Users
+should be able to directly read the HDFS files in the Hive tables using other
+tools, or use other tools to directly write to HDFS files that can be read by
+Hive through "CREATE EXTERNAL TABLE", or can be loaded into Hive through "LOAD
+DATA INPATH" which just move the file into Hive table directory.
+
+
+Note that org.apache.hadoop.hive.serde is the deprecated old serde library.
+Please look at org.apache.hadoop.hive.serde2 for the latest version.
+
+
+Existing FileFormats and SerDe classes
+------------------------
+Hive currently use these FileFormats to read/write to files:
+
+* TextInputFormat/NoKeyTextOutputFormat
+  These 2 classes read/write data in plain text file format.
+
+* SequenceFileInputFormat/SequenceFileOutputFormat
+  These 2 classes read/write data in hadoop SequenceFile format.
+
+Hive currently use these SerDe classes to serialize and deserialize data:
+
+* MetadataTypedColumnsetSerDe
+  This serde is used to read/write delimited records like CSV, tab-separated
+  control-A separated records (sorry, quote is not supported yet.)
+
+* ThriftSerDe
+  This serde is used to read/write thrift serialized objects.  The class file
+  for the Thrift object must be loaded first.
+
+* DynamicSerDe
+  This serde also read/write thrift serialized objects, but it understands thrift
+  DDL so the schema of the object can be provided at runtime.  Also it supports
+  a lot of different protocols, including TBinaryProtocol, TJSONProtocol,
+  TCTLSeparatedProtocol (which writes data in delimited records).
+
+
+
+How to load data into Hive
+------------------------
+In order to load data into Hive, we need to tell Hive the format of the data
+through "CREATE TABLE" statement:
+
+* FileFormat: the data has to be in Text or SequenceFile.
+* Format of the row:
+  * If the data is in delimited format, use MetadataTypedColumnsetSerDe
+  * If the data is in delimited format and has more than 1 levels of delimitor,
+    use DynamicSerDe with TCTLSeparatedProtocol
+  * If the data is a serialized thrift object, use ThriftSerDe
+
+The steps to load the data:
+1 Create a table:
+
+  CREATE TABLE t (foo STRING, bar STRING)
+  ROW FORMAT DELIMITED
+  FIELDS TERMINATED BY '\t'
+  STORED AS TEXTFILE;
+
+  CREATE TABLE t2 (foo STRING, bar ARRAY<STRING>)
+  ROW FORMAT DELIMITED
+  FIELDS TERMINATED BY '\t'
+  COLLECTION ITEMS TERMINATED BY ','
+  STORED AS TEXTFILE;
+
+  CREATE TABLE t3 (foo STRING, bar MAP<STRING,STRING>)
+  ROW FORMAT DELIMITED
+  FIELDS TERMINATED BY '\t'
+  COLLECTION ITEMS TERMINATED BY ','
+  MAP KEYS TERMINATED BY ':'
+  STORED AS TEXTFILE;
+
+  CREATE TABLE t4 (foo STRING, bar MAP<STRING,STRING>)
+  ROW FORMAT SERIALIZER 'org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe'
+  WITH SERDEPROPERTIES ('columns'='foo,bar','SERIALIZATION.FORMAT'='9');
+
+  (RegexDeserializer is not done yet)
+  CREATE TABLE t5 (foo STRING, bar STRING)
+  ROW FORMAT SERIALIZER 'org.apache.hadoop.hive.serde2.RegexDeserializer'
+  WITH SERDEPROPERTIES ('regex'='([a-z]*) *([a-z]*)');
+
+2 Load the data:
+  LOAD DATA LOCAL INPATH '../examples/files/kv1.txt' OVERWRITE INTO TABLE t;
+
+
+
+How to read data from Hive tables
+------------------------
+In order to read data from Hive tables, we need to know the same 2 things:
+* File Format
+* Row Format
+
+Then we just need to directly open the HDFS file and read the data.
+
+
+How to write your own SerDe
+------------------------
+
+In most cases, users want to write a Deserializer instead of a SerDe.
+For example, the RegexDeserializer will deserialize the data using the
+configuration parameter 'regex', and possibly a list of column names (see
+serde2.MetadataTypedColumnsetSerDe).
+
+Please see serde2/Deserializer.java for details.

Modified: hadoop/core/trunk/src/contrib/hive/serde/build.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/build.xml?rev=712905&r1=712904&r2=712905&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/build.xml (original)
+++ hadoop/core/trunk/src/contrib/hive/serde/build.xml Mon Nov 10 17:50:06 2008
@@ -27,21 +27,27 @@
   <property name="src.dir"  location="${basedir}/src"/> 
   <import file="../build-common.xml"/>
 
+
   <target name="dynamic-serde" depends="init" if="javacc.home">
-    <mkdir dir="${src.dir}/gen-java/org/apache/hadoop/hive/serde2/dynamic_type"/>
+
+<!--    <mkdir dir="${src.dir}/gen-java/org/apache/hadoop/hive/serde2/dynamic_type"/> -->
+    <jjtree buildnodefiles="true"
+      target="${src.dir}/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammar.jjt"
+      outputdirectory="${src.dir}/java/org/apache/hadoop/hive/serde2/dynamic_type"
+      javacchome="${javacc.home}"
+    />
     <javacc
-      target="${src.dir}/gen-java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammar.jj"
-      outputdirectory="${src.dir}/gen-java/org/apache/hadoop/hive/serde2/dynamic_type"
+      target="${src.dir}/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammar.jj"
+      outputdirectory="${src.dir}/java/org/apache/hadoop/hive/serde2/dynamic_type"
       javacchome="${javacc.home}"
     />
   </target>
 
-  <target name="compile" depends="init, dynamic-serde">
+  <target name="compile" depends="init,dynamic-serde">
     <echo message="Compiling: ${name}"/>
     <javac
      encoding="${build.encoding}"
-     srcdir="${src.dir}/java:${src.dir}/gen-java"
-     includes="**/*.java"
+     srcdir="${src.dir}/java/:${src.dir}/gen-java/"
      destdir="${build.classes}"
      debug="${javac.debug}"
      deprecation="${javac.deprecation}">

Modified: hadoop/core/trunk/src/contrib/hive/serde/if/serde.thrift
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/if/serde.thrift?rev=712905&r1=712904&r2=712905&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/if/serde.thrift (original)
+++ hadoop/core/trunk/src/contrib/hive/serde/if/serde.thrift Mon Nov 10 17:50:06 2008
@@ -9,20 +9,25 @@
 const string SERIALIZATION_FORMAT = "serialization.format"
 const string SERIALIZATION_DDL = "serialization.ddl"
 const string SERIALIZATION_NULL_FORMAT = "serialization.null.format"
+const string SERIALIZATION_LAST_COLUMN_TAKES_REST = "serialization.last.column.takes.rest"
+const string SERIALIZATION_SORT_ORDER = "serialization.sort.order"
 
 const string FIELD_DELIM = "field.delim"
 const string COLLECTION_DELIM = "colelction.delim"
 const string LINE_DELIM = "line.delim"
 const string MAPKEY_DELIM = "mapkey.delim"
+const string QUOTE_CHAR = "quote.delim"
 
 typedef string PrimitiveType
 typedef string CollectionType
 
+const string BOOLEAN_TYPE_NAME  = "boolean";
 const string TINYINT_TYPE_NAME   = "tinyint";
+const string SMALLINT_TYPE_NAME  = "smallint";
 const string INT_TYPE_NAME       = "int";
 const string BIGINT_TYPE_NAME    = "bigint";
 const string FLOAT_TYPE_NAME     = "float";
-const string DOUBLE_TYPE_NAME    = "double"; 
+const string DOUBLE_TYPE_NAME    = "double";
 const string STRING_TYPE_NAME    = "string";
 const string DATE_TYPE_NAME      = "date";
 const string DATETIME_TYPE_NAME  = "datetime";
@@ -31,7 +36,7 @@
 const string LIST_TYPE_NAME = "array";
 const string MAP_TYPE_NAME  = "map";
 
-const set<string> PrimitiveTypes  = [ TINYINT_TYPE_NAME INT_TYPE_NAME BIGINT_TYPE_NAME FLOAT_TYPE_NAME DOUBLE_TYPE_NAME STRING_TYPE_NAME  DATE_TYPE_NAME DATETIME_TYPE_NAME TIMESTAMP_TYPE_NAME ],
+const set<string> PrimitiveTypes  = [ BOOLEAN_TYPE_NAME TINYINT_TYPE_NAME SMALLINT_TYPE_NAME INT_TYPE_NAME BIGINT_TYPE_NAME FLOAT_TYPE_NAME DOUBLE_TYPE_NAME STRING_TYPE_NAME  DATE_TYPE_NAME DATETIME_TYPE_NAME TIMESTAMP_TYPE_NAME ],
 const set<string> CollectionTypes = [ LIST_TYPE_NAME MAP_TYPE_NAME ],
 
 

Modified: hadoop/core/trunk/src/contrib/hive/serde/src/gen-java/org/apache/hadoop/hive/serde/Constants.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/gen-java/org/apache/hadoop/hive/serde/Constants.java?rev=712905&r1=712904&r2=712905&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/gen-java/org/apache/hadoop/hive/serde/Constants.java (original)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/gen-java/org/apache/hadoop/hive/serde/Constants.java Mon Nov 10 17:50:06 2008
@@ -25,6 +25,10 @@
 
   public static final String SERIALIZATION_NULL_FORMAT = "serialization.null.format";
 
+  public static final String SERIALIZATION_LAST_COLUMN_TAKES_REST = "serialization.last.column.takes.rest";
+
+  public static final String SERIALIZATION_SORT_ORDER = "serialization.sort.order";
+
   public static final String FIELD_DELIM = "field.delim";
 
   public static final String COLLECTION_DELIM = "colelction.delim";
@@ -33,8 +37,14 @@
 
   public static final String MAPKEY_DELIM = "mapkey.delim";
 
+  public static final String QUOTE_CHAR = "quote.delim";
+
+  public static final String BOOLEAN_TYPE_NAME = "boolean";
+
   public static final String TINYINT_TYPE_NAME = "tinyint";
 
+  public static final String SMALLINT_TYPE_NAME = "smallint";
+
   public static final String INT_TYPE_NAME = "int";
 
   public static final String BIGINT_TYPE_NAME = "bigint";
@@ -57,7 +67,9 @@
 
   public static final Set<String> PrimitiveTypes = new HashSet<String>();
   static {
+    PrimitiveTypes.add("boolean");
     PrimitiveTypes.add("tinyint");
+    PrimitiveTypes.add("smallint");
     PrimitiveTypes.add("int");
     PrimitiveTypes.add("bigint");
     PrimitiveTypes.add("float");

Modified: hadoop/core/trunk/src/contrib/hive/serde/src/gen-php/serde_constants.php
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/gen-php/serde_constants.php?rev=712905&r1=712904&r2=712905&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/gen-php/serde_constants.php (original)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/gen-php/serde_constants.php Mon Nov 10 17:50:06 2008
@@ -16,6 +16,12 @@
 
 $GLOBALS['serde_CONSTANTS']['SERIALIZATION_DDL'] = 'serialization.ddl';
 
+$GLOBALS['serde_CONSTANTS']['SERIALIZATION_NULL_FORMAT'] = 'serialization.null.format';
+
+$GLOBALS['serde_CONSTANTS']['SERIALIZATION_LAST_COLUMN_TAKES_REST'] = 'serialization.last.column.takes.rest';
+
+$GLOBALS['serde_CONSTANTS']['SERIALIZATION_SORT_ORDER'] = 'serialization.sort.order';
+
 $GLOBALS['serde_CONSTANTS']['FIELD_DELIM'] = 'field.delim';
 
 $GLOBALS['serde_CONSTANTS']['COLLECTION_DELIM'] = 'colelction.delim';
@@ -24,8 +30,14 @@
 
 $GLOBALS['serde_CONSTANTS']['MAPKEY_DELIM'] = 'mapkey.delim';
 
+$GLOBALS['serde_CONSTANTS']['QUOTE_CHAR'] = 'quote.delim';
+
+$GLOBALS['serde_CONSTANTS']['BOOLEAN_TYPE_NAME'] = 'boolean';
+
 $GLOBALS['serde_CONSTANTS']['TINYINT_TYPE_NAME'] = 'tinyint';
 
+$GLOBALS['serde_CONSTANTS']['SMALLINT_TYPE_NAME'] = 'smallint';
+
 $GLOBALS['serde_CONSTANTS']['INT_TYPE_NAME'] = 'int';
 
 $GLOBALS['serde_CONSTANTS']['BIGINT_TYPE_NAME'] = 'bigint';
@@ -47,7 +59,9 @@
 $GLOBALS['serde_CONSTANTS']['MAP_TYPE_NAME'] = 'map';
 
 $GLOBALS['serde_CONSTANTS']['PrimitiveTypes'] = array(
+  'boolean' => true,
   'tinyint' => true,
+  'smallint' => true,
   'int' => true,
   'bigint' => true,
   'float' => true,

Modified: hadoop/core/trunk/src/contrib/hive/serde/src/gen-py/org_apache_hadoop_hive_serde/constants.py
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/gen-py/org_apache_hadoop_hive_serde/constants.py?rev=712905&r1=712904&r2=712905&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/gen-py/org_apache_hadoop_hive_serde/constants.py (original)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/gen-py/org_apache_hadoop_hive_serde/constants.py Mon Nov 10 17:50:06 2008
@@ -15,6 +15,12 @@
 
 SERIALIZATION_DDL = 'serialization.ddl'
 
+SERIALIZATION_NULL_FORMAT = 'serialization.null.format'
+
+SERIALIZATION_LAST_COLUMN_TAKES_REST = 'serialization.last.column.takes.rest'
+
+SERIALIZATION_SORT_ORDER = 'serialization.sort.order'
+
 FIELD_DELIM = 'field.delim'
 
 COLLECTION_DELIM = 'colelction.delim'
@@ -23,8 +29,14 @@
 
 MAPKEY_DELIM = 'mapkey.delim'
 
+QUOTE_CHAR = 'quote.delim'
+
+BOOLEAN_TYPE_NAME = 'boolean'
+
 TINYINT_TYPE_NAME = 'tinyint'
 
+SMALLINT_TYPE_NAME = 'smallint'
+
 INT_TYPE_NAME = 'int'
 
 BIGINT_TYPE_NAME = 'bigint'
@@ -46,7 +58,9 @@
 MAP_TYPE_NAME = 'map'
 
 PrimitiveTypes = set([
+  'boolean',
   'tinyint',
+  'smallint',
   'int',
   'bigint',
   'float',

Modified: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde/SerDeUtils.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde/SerDeUtils.java?rev=712905&r1=712904&r2=712905&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde/SerDeUtils.java (original)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde/SerDeUtils.java Mon Nov 10 17:50:06 2008
@@ -64,7 +64,7 @@
     // Eagerly load SerDes so they will register their symbolic names even on Lazy Loading JVMs
     try {
       // loading these classes will automatically register the short names
-      Class.forName(org.apache.hadoop.hive.serde.dynamic_type.DynamicSerDe.class.getName());
+      Class.forName(org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe.class.getName());
       Class.forName(org.apache.hadoop.hive.serde.jute.JuteSerDe.class.getName());
       Class.forName(org.apache.hadoop.hive.serde.simple_meta.MetadataTypedColumnsetSerDe.class.getName());
       Class.forName(org.apache.hadoop.hive.serde.thrift.columnsetSerDe.class.getName());

Modified: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde/simple_meta/MetadataTypedColumnsetSerDe.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde/simple_meta/MetadataTypedColumnsetSerDe.java?rev=712905&r1=712904&r2=712905&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde/simple_meta/MetadataTypedColumnsetSerDe.java (original)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde/simple_meta/MetadataTypedColumnsetSerDe.java Mon Nov 10 17:50:06 2008
@@ -18,17 +18,36 @@
 
 package org.apache.hadoop.hive.serde.simple_meta;
 
-import org.apache.hadoop.hive.serde.*;
-import org.apache.hadoop.hive.serde.thrift.*;
-import com.facebook.thrift.TException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.serde.ByteStreamTypedSerDe;
+import org.apache.hadoop.hive.serde.ColumnSet;
+import org.apache.hadoop.hive.serde.ComplexSerDeField;
+import org.apache.hadoop.hive.serde.Constants;
+import org.apache.hadoop.hive.serde.ExpressionUtils;
+import org.apache.hadoop.hive.serde.ReflectionSerDeField;
+import org.apache.hadoop.hive.serde.SerDe;
+import org.apache.hadoop.hive.serde.SerDeException;
+import org.apache.hadoop.hive.serde.SerDeField;
+import org.apache.hadoop.hive.serde.SerDeUtils;
+import org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.util.StringUtils;
+
 import com.facebook.thrift.TBase;
+import com.facebook.thrift.TException;
 import com.facebook.thrift.TSerializer;
-import com.facebook.thrift.protocol.*;
-import com.facebook.thrift.transport.*;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.*;
-import java.util.*;
-import java.io.*;
+import com.facebook.thrift.protocol.TBinaryProtocol;
+import com.facebook.thrift.protocol.TProtocol;
+import com.facebook.thrift.protocol.TProtocolFactory;
+import com.facebook.thrift.protocol.TSimpleJSONProtocol;
+import com.facebook.thrift.transport.TIOStreamTransport;
 
 
 
@@ -51,6 +70,18 @@
   // stores the columns in order
   private String _columns_list[];
 
+  static {
+    StackTraceElement[] sTrace = new Exception().getStackTrace();
+    String className = sTrace[0].getClassName();
+    try {
+      // For backward compatibility: this class replaces the columnsetSerDe class.
+      SerDeUtils.registerSerDe(DynamicSerDe.class.getName(), 
+          Class.forName(className));
+      } catch(Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+  
   public String toString() {
     return "MetaDataTypedColumnsetSerDe[" + separator + "," + _columns + "]";
   }
@@ -79,7 +110,12 @@
   public void initialize(Configuration job, Properties tbl) throws SerDeException {
     inStreaming = job.get("hive.streaming.select") != null;
     separator = DefaultSeparator;
-    String alt_sep = tbl.getProperty(Constants.SERIALIZATION_FORMAT);
+    String alt_sep = null;
+    if(DynamicSerDe.class.getName().equals(tbl.getProperty(Constants.SERIALIZATION_LIB))) {
+      alt_sep = tbl.getProperty(Constants.FIELD_DELIM);
+    } else {
+      alt_sep = tbl.getProperty(Constants.SERIALIZATION_FORMAT);
+    }
     if(alt_sep != null && alt_sep.length() > 0) {
       try {
         byte b [] = new byte[1];
@@ -125,8 +161,13 @@
     ColumnSet c = cachedObj;
     try {
       try {
-        Text tw = (Text)field;
-        String row = tw.toString();
+        String row = null;
+        if(field instanceof BytesWritable) {
+          row = new String(((BytesWritable) field).get(), 0, ((BytesWritable) field).getSize(), "UTF-8");
+        } else {
+          Text tw = (Text)field;
+          row = tw.toString();
+        }
         return(deserialize(c, row, separator, nullString));
       } catch (ClassCastException e) {
         throw new SerDeException("columnsetSerDe  expects Text", e);

Modified: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/MetadataTypedColumnsetSerDe.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/MetadataTypedColumnsetSerDe.java?rev=712905&r1=712904&r2=712905&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/MetadataTypedColumnsetSerDe.java (original)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/MetadataTypedColumnsetSerDe.java Mon Nov 10 17:50:06 2008
@@ -64,6 +64,9 @@
   private List<String> columnNames;
   private ObjectInspector cachedObjectInspector;
 
+  private boolean lastColumnTakesRest = false;
+  private int splitLimit = -1;
+  
   public String toString() {
     return "MetaDataTypedColumnsetSerDe[" + separator + "," + columnNames + "]";
   }
@@ -103,23 +106,36 @@
     if (columnProperty == null || columnProperty.length() == 0 
         || columnsetSerDe) {
       // Hack for tables with no columns
-      // Treat it as a table with a single column called "col" 
+      // Treat it as a table with a single column called "col"
       cachedObjectInspector = ObjectInspectorFactory.getReflectionObjectInspector(
           ColumnSet.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
     } else {
       columnNames = Arrays.asList(columnProperty.split(","));
       cachedObjectInspector = MetadataListStructObjectInspector.getInstance(columnNames);
     }
-    LOG.debug(getClass().getName() + ": initialized with columnNames: " + columnNames + " and separator code=" + (int)separator.charAt(0) );
+    
+    String lastColumnTakesRestString = tbl.getProperty(Constants.SERIALIZATION_LAST_COLUMN_TAKES_REST);
+    lastColumnTakesRest = (lastColumnTakesRestString != null && lastColumnTakesRestString.equalsIgnoreCase("true"));
+    splitLimit = (lastColumnTakesRest && columnNames != null) ? columnNames.size() : -1; 
+    
+    LOG.debug(getClass().getName() + ": initialized with columnNames: " + columnNames + " and separator code=" + (int)separator.charAt(0) 
+        + " lastColumnTakesRest=" + lastColumnTakesRest + " splitLimit=" + splitLimit);
   }
   
-  public static Object deserialize(ColumnSet c, String row, String sep, String nullString) throws Exception {
+  /**
+   * Split the row into columns.
+   * @param limit  up to limit columns will be produced (the last column takes all the rest), -1 for unlimited.
+   * @return
+   * @throws Exception
+   */
+  public static Object deserialize(ColumnSet c, String row, String sep,
+      String nullString, int limit) throws Exception {
     if (c.col == null) {
       c.col = new ArrayList<String>();
     } else {
       c.col.clear();
     }
-    String [] l1 = row.split(sep, -1);
+    String [] l1 = row.split(sep, limit);
 
     for(String s: l1) {
       if (s.equals(nullString)) {
@@ -145,7 +161,7 @@
       row = field.toString();
     }
     try {
-      deserialize(deserializeCache, row, separator, nullString);
+      deserialize(deserializeCache, row, separator, nullString, splitLimit);
       if (columnNames != null) {
         assert(columnNames.size() == deserializeCache.col.size());
       }

Modified: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java?rev=712905&r1=712904&r2=712905&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java (original)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java Mon Nov 10 17:50:06 2008
@@ -64,6 +64,17 @@
     }
   }
 
+  private static List<String> nativeSerDeNames = new ArrayList<String>(); 
+  static {
+    nativeSerDeNames.add(org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe.class.getName());
+    nativeSerDeNames.add(org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe.class.getName());
+    nativeSerDeNames.add(org.apache.hadoop.hive.serde.thrift.columnsetSerDe.class.getName());
+  }
+
+  public static boolean isNativeSerDe(String serde) {
+    return nativeSerDeNames.contains(serde);
+  }
+  
 
   private static boolean initCoreSerDes = registerCoreSerDes();
   

Modified: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDe.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDe.java?rev=712905&r1=712904&r2=712905&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDe.java (original)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDe.java Mon Nov 10 17:50:06 2008
@@ -108,7 +108,7 @@
 
       this.bt.initialize();
     } catch (Exception e) {
-      System.out.println(StringUtils.stringifyException(e));
+      System.err.println(StringUtils.stringifyException(e));
       throw new SerDeException(e);
     }
   }

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeAsync.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeAsync.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeAsync.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeAsync.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,14 @@
+/* Generated By:JJTree: Do not edit this line. DynamicSerDeAsync.java */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+public class DynamicSerDeAsync extends SimpleNode {
+  public DynamicSerDeAsync(int id) {
+    super(id);
+  }
+
+  public DynamicSerDeAsync(thrift_grammar p, int id) {
+    super(p, id);
+  }
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeCommaOrSemicolon.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeCommaOrSemicolon.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeCommaOrSemicolon.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeCommaOrSemicolon.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,14 @@
+/* Generated By:JJTree: Do not edit this line. DynamicSerDeCommaOrSemicolon.java */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+public class DynamicSerDeCommaOrSemicolon extends SimpleNode {
+  public DynamicSerDeCommaOrSemicolon(int id) {
+    super(id);
+  }
+
+  public DynamicSerDeCommaOrSemicolon(thrift_grammar p, int id) {
+    super(p, id);
+  }
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeConst.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeConst.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeConst.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeConst.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,14 @@
+/* Generated By:JJTree: Do not edit this line. DynamicSerDeConst.java */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+public class DynamicSerDeConst extends SimpleNode {
+  public DynamicSerDeConst(int id) {
+    super(id);
+  }
+
+  public DynamicSerDeConst(thrift_grammar p, int id) {
+    super(p, id);
+  }
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeConstList.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeConstList.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeConstList.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeConstList.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,14 @@
+/* Generated By:JJTree: Do not edit this line. DynamicSerDeConstList.java */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+public class DynamicSerDeConstList extends SimpleNode {
+  public DynamicSerDeConstList(int id) {
+    super(id);
+  }
+
+  public DynamicSerDeConstList(thrift_grammar p, int id) {
+    super(p, id);
+  }
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeConstListContents.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeConstListContents.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeConstListContents.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeConstListContents.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,14 @@
+/* Generated By:JJTree: Do not edit this line. DynamicSerDeConstListContents.java */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+public class DynamicSerDeConstListContents extends SimpleNode {
+  public DynamicSerDeConstListContents(int id) {
+    super(id);
+  }
+
+  public DynamicSerDeConstListContents(thrift_grammar p, int id) {
+    super(p, id);
+  }
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeConstMap.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeConstMap.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeConstMap.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeConstMap.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,14 @@
+/* Generated By:JJTree: Do not edit this line. DynamicSerDeConstMap.java */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+public class DynamicSerDeConstMap extends SimpleNode {
+  public DynamicSerDeConstMap(int id) {
+    super(id);
+  }
+
+  public DynamicSerDeConstMap(thrift_grammar p, int id) {
+    super(p, id);
+  }
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeConstMapContents.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeConstMapContents.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeConstMapContents.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeConstMapContents.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,14 @@
+/* Generated By:JJTree: Do not edit this line. DynamicSerDeConstMapContents.java */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+public class DynamicSerDeConstMapContents extends SimpleNode {
+  public DynamicSerDeConstMapContents(int id) {
+    super(id);
+  }
+
+  public DynamicSerDeConstMapContents(thrift_grammar p, int id) {
+    super(p, id);
+  }
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeConstValue.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeConstValue.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeConstValue.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeConstValue.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,14 @@
+/* Generated By:JJTree: Do not edit this line. DynamicSerDeConstValue.java */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+public class DynamicSerDeConstValue extends SimpleNode {
+  public DynamicSerDeConstValue(int id) {
+    super(id);
+  }
+
+  public DynamicSerDeConstValue(thrift_grammar p, int id) {
+    super(p, id);
+  }
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeDefinition.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeDefinition.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeDefinition.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeDefinition.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,14 @@
+/* Generated By:JJTree: Do not edit this line. DynamicSerDeDefinition.java */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+public class DynamicSerDeDefinition extends SimpleNode {
+  public DynamicSerDeDefinition(int id) {
+    super(id);
+  }
+
+  public DynamicSerDeDefinition(thrift_grammar p, int id) {
+    super(p, id);
+  }
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeDefinitionType.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeDefinitionType.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeDefinitionType.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeDefinitionType.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,14 @@
+/* Generated By:JJTree: Do not edit this line. DynamicSerDeDefinitionType.java */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+public class DynamicSerDeDefinitionType extends SimpleNode {
+  public DynamicSerDeDefinitionType(int id) {
+    super(id);
+  }
+
+  public DynamicSerDeDefinitionType(thrift_grammar p, int id) {
+    super(p, id);
+  }
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeEnum.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeEnum.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeEnum.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeEnum.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,14 @@
+/* Generated By:JJTree: Do not edit this line. DynamicSerDeEnum.java */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+public class DynamicSerDeEnum extends SimpleNode {
+  public DynamicSerDeEnum(int id) {
+    super(id);
+  }
+
+  public DynamicSerDeEnum(thrift_grammar p, int id) {
+    super(p, id);
+  }
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeEnumDef.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeEnumDef.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeEnumDef.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeEnumDef.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,14 @@
+/* Generated By:JJTree: Do not edit this line. DynamicSerDeEnumDef.java */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+public class DynamicSerDeEnumDef extends SimpleNode {
+  public DynamicSerDeEnumDef(int id) {
+    super(id);
+  }
+
+  public DynamicSerDeEnumDef(thrift_grammar p, int id) {
+    super(p, id);
+  }
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeEnumDefList.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeEnumDefList.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeEnumDefList.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeEnumDefList.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,14 @@
+/* Generated By:JJTree: Do not edit this line. DynamicSerDeEnumDefList.java */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+public class DynamicSerDeEnumDefList extends SimpleNode {
+  public DynamicSerDeEnumDefList(int id) {
+    super(id);
+  }
+
+  public DynamicSerDeEnumDefList(thrift_grammar p, int id) {
+    super(p, id);
+  }
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeExtends.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeExtends.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeExtends.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeExtends.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,14 @@
+/* Generated By:JJTree: Do not edit this line. DynamicSerDeExtends.java */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+public class DynamicSerDeExtends extends SimpleNode {
+  public DynamicSerDeExtends(int id) {
+    super(id);
+  }
+
+  public DynamicSerDeExtends(thrift_grammar p, int id) {
+    super(p, id);
+  }
+
+}

Modified: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeField.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeField.java?rev=712905&r1=712904&r2=712905&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeField.java (original)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeField.java Mon Nov 10 17:50:06 2008
@@ -32,6 +32,10 @@
   private final int FD_FIELD_TYPE = 1;
   private final int FD_FIELD_VALUE =2;
 
+  public boolean isSkippable() {
+    return ((DynamicSerDeFieldRequiredness)this.jjtGetChild(FD_REQUIREDNESS)).getRequiredness() == DynamicSerDeFieldRequiredness.RequirednessTypes.Skippable;
+  }
+
   public DynamicSerDeFieldType getFieldType() {
     return (DynamicSerDeFieldType)this.jjtGetChild(FD_FIELD_TYPE);
   }

Modified: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldList.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldList.java?rev=712905&r1=712904&r2=712905&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldList.java (original)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldList.java Mon Nov 10 17:50:06 2008
@@ -87,6 +87,9 @@
       for(int i = 0 ; i < this.jjtGetNumChildren(); i++) {
         DynamicSerDeField mt = this.getField(i);
         DynamicSerDeTypeBase type = mt.getFieldType().getMyType();
+        // types get initialized in case they need to setup any 
+        // internal data structures - e.g., DynamicSerDeStructBase
+        type.initialize();
         type.fieldid = mt.fieldid;
         type.name = mt.name;
 
@@ -106,6 +109,15 @@
     return types_by_column_name.get(fieldname);
   }
 
+  /**
+   * Indicates whether fields can be out of order or missing. i.e., is it really real
+   * thrift serialization.
+   * This is used by dynamicserde to do some optimizations if it knows all the fields exist
+   * and are required and are serialized in order.
+   * For now, those optimizations are only done for DynamicSerDe serialized data so always
+   * set to false for now.
+   */
+  protected boolean isRealThrift = false;
 
   public Object deserialize(Object reuse, TProtocol iprot)  throws SerDeException, TException, IllegalAccessException {
     ArrayList<Object> struct = null;
@@ -120,16 +132,33 @@
       assert(struct.size() == this.getNumFields());
     }
 
+    boolean fastSkips = iprot instanceof org.apache.hadoop.hive.serde2.thrift.SkippableTProtocol;
+
+    // may need to strip away the STOP marker when in thrift mode
+    boolean stopSeen = false;
+
     // Read the fields.
     for(int i = 0; i < this.getNumFields(); i++) {
       DynamicSerDeTypeBase mt = null;
       TField field = null;
-
+      
+      if(!isRealThrift && this.getField(i).isSkippable()) {
+        // PRE - all the fields are required and serialized in order - is !isRealThrift
+        mt = this.ordered_types[i];
+        if(fastSkips) {
+          ((org.apache.hadoop.hive.serde2.thrift.SkippableTProtocol)iprot).skip(mt.getType());
+        } else {
+          TProtocolUtil.skip(iprot,mt.getType());
+        }
+        struct.set(i, null);
+        continue;
+      }
       if (thrift_mode) {
         field = iprot.readFieldBegin();
 
         if(field.type >= 0) {
           if(field.type == TType.STOP) {
+            stopSeen = true;
             break;
           }
           mt = this.getFieldByFieldId(field.id);
@@ -157,23 +186,28 @@
         orderedId = ordered_column_id_by_name.get(mt.name); 
       }
       struct.set(orderedId, mt.deserialize(struct.get(orderedId), iprot));
-
       if(thrift_mode) {
         iprot.readFieldEnd();
       }
     }
+    if(thrift_mode && !stopSeen) {
+      // strip off the STOP marker, which may be left if all the fields were in the serialization
+      TField field = iprot.readFieldBegin();
+      assert(field.type == TType.STOP);
+    }
     return struct;
   }
 
 
   TField field = new TField();
   public void serialize(Object o, ObjectInspector oi, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException,IllegalAccessException  {
-
     // Assuming the ObjectInspector represents exactly the same type as this struct.
     // This assumption should be checked during query compile time.
     assert(oi instanceof StructObjectInspector);
     StructObjectInspector soi = (StructObjectInspector) oi;
 
+    boolean writeNulls = oprot instanceof org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol;
+    
     // For every field
     List<? extends StructField> fields = soi.getAllStructFieldRefs();
     if (fields.size() != ordered_types.length) {
@@ -184,6 +218,10 @@
       Object f = soi.getStructFieldData(o, fields.get(i));
       DynamicSerDeTypeBase mt = ordered_types[i];
       
+      if (f == null && !writeNulls) {
+        continue;
+      }
+
       if(thrift_mode) {
         field.name = mt.name;
         field.type = mt.getType();
@@ -191,8 +229,11 @@
         oprot.writeFieldBegin(field);
       }
 
-      mt.serialize(f, fields.get(i).getFieldObjectInspector(), oprot);
-
+      if(f == null) {
+        ((org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol)oprot).writeNull();
+      } else {
+        mt.serialize(f, fields.get(i).getFieldObjectInspector(), oprot);
+      }
       if(thrift_mode) {
         oprot.writeFieldEnd();
       }

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldRequiredness.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldRequiredness.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldRequiredness.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldRequiredness.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+public class DynamicSerDeFieldRequiredness extends SimpleNode {
+  public enum RequirednessTypes 
+  {
+    Required,  Skippable, Optional,
+      };
+
+  /**
+   *  Is this a required, skippable or optional field.
+   *  Used by DynamicSerDe for optimizations.
+   */
+  protected RequirednessTypes requiredness;
+
+  /**
+   * Get the requiredness attribute of this field.
+   */
+  public RequirednessTypes getRequiredness() {
+    return requiredness;
+  }
+
+  public DynamicSerDeFieldRequiredness(int id) {
+    super(id);
+  }
+
+  public DynamicSerDeFieldRequiredness(thrift_grammar p, int id) {
+    super(p, id);
+  }
+
+}

Modified: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldType.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldType.java?rev=712905&r1=712904&r2=712905&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldType.java (original)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldType.java Mon Nov 10 17:50:06 2008
@@ -37,6 +37,4 @@
     // and thus we can quickly find this comment and limitation.
     return (DynamicSerDeTypeBase)this.jjtGetChild(FD_FIELD_TYPE);
   }
-
-
 }

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldValue.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldValue.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldValue.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldValue.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,14 @@
+/* Generated By:JJTree: Do not edit this line. DynamicSerDeFieldValue.java */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+public class DynamicSerDeFieldValue extends SimpleNode {
+  public DynamicSerDeFieldValue(int id) {
+    super(id);
+  }
+
+  public DynamicSerDeFieldValue(thrift_grammar p, int id) {
+    super(p, id);
+  }
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFlagArgs.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFlagArgs.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFlagArgs.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFlagArgs.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,14 @@
+/* Generated By:JJTree: Do not edit this line. DynamicSerDeFlagArgs.java */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+public class DynamicSerDeFlagArgs extends SimpleNode {
+  public DynamicSerDeFlagArgs(int id) {
+    super(id);
+  }
+
+  public DynamicSerDeFlagArgs(thrift_grammar p, int id) {
+    super(p, id);
+  }
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFunctionType.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFunctionType.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFunctionType.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFunctionType.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,14 @@
+/* Generated By:JJTree: Do not edit this line. DynamicSerDeFunctionType.java */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+public class DynamicSerDeFunctionType extends SimpleNode {
+  public DynamicSerDeFunctionType(int id) {
+    super(id);
+  }
+
+  public DynamicSerDeFunctionType(thrift_grammar p, int id) {
+    super(p, id);
+  }
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeHeader.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeHeader.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeHeader.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeHeader.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,14 @@
+/* Generated By:JJTree: Do not edit this line. DynamicSerDeHeader.java */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+public class DynamicSerDeHeader extends SimpleNode {
+  public DynamicSerDeHeader(int id) {
+    super(id);
+  }
+
+  public DynamicSerDeHeader(thrift_grammar p, int id) {
+    super(p, id);
+  }
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeHeaderList.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeHeaderList.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeHeaderList.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeHeaderList.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,14 @@
+/* Generated By:JJTree: Do not edit this line. DynamicSerDeHeaderList.java */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+public class DynamicSerDeHeaderList extends SimpleNode {
+  public DynamicSerDeHeaderList(int id) {
+    super(id);
+  }
+
+  public DynamicSerDeHeaderList(thrift_grammar p, int id) {
+    super(p, id);
+  }
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeInclude.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeInclude.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeInclude.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeInclude.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,14 @@
+/* Generated By:JJTree: Do not edit this line. DynamicSerDeInclude.java */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+public class DynamicSerDeInclude extends SimpleNode {
+  public DynamicSerDeInclude(int id) {
+    super(id);
+  }
+
+  public DynamicSerDeInclude(thrift_grammar p, int id) {
+    super(p, id);
+  }
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeNamespace.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeNamespace.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeNamespace.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeNamespace.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,14 @@
+/* Generated By:JJTree: Do not edit this line. DynamicSerDeNamespace.java */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+public class DynamicSerDeNamespace extends SimpleNode {
+  public DynamicSerDeNamespace(int id) {
+    super(id);
+  }
+
+  public DynamicSerDeNamespace(thrift_grammar p, int id) {
+    super(p, id);
+  }
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeSenum.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeSenum.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeSenum.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeSenum.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,14 @@
+/* Generated By:JJTree: Do not edit this line. DynamicSerDeSenum.java */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+public class DynamicSerDeSenum extends SimpleNode {
+  public DynamicSerDeSenum(int id) {
+    super(id);
+  }
+
+  public DynamicSerDeSenum(thrift_grammar p, int id) {
+    super(p, id);
+  }
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeSenumDef.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeSenumDef.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeSenumDef.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeSenumDef.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,14 @@
+/* Generated By:JJTree: Do not edit this line. DynamicSerDeSenumDef.java */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+public class DynamicSerDeSenumDef extends SimpleNode {
+  public DynamicSerDeSenumDef(int id) {
+    super(id);
+  }
+
+  public DynamicSerDeSenumDef(thrift_grammar p, int id) {
+    super(p, id);
+  }
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeSenumDefList.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeSenumDefList.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeSenumDefList.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeSenumDefList.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,14 @@
+/* Generated By:JJTree: Do not edit this line. DynamicSerDeSenumDefList.java */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+public class DynamicSerDeSenumDefList extends SimpleNode {
+  public DynamicSerDeSenumDefList(int id) {
+    super(id);
+  }
+
+  public DynamicSerDeSenumDefList(thrift_grammar p, int id) {
+    super(p, id);
+  }
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeService.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeService.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeService.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeService.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,14 @@
+/* Generated By:JJTree: Do not edit this line. DynamicSerDeService.java */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+public class DynamicSerDeService extends SimpleNode {
+  public DynamicSerDeService(int id) {
+    super(id);
+  }
+
+  public DynamicSerDeService(thrift_grammar p, int id) {
+    super(p, id);
+  }
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeStart.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeStart.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeStart.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeStart.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,14 @@
+/* Generated By:JJTree: Do not edit this line. DynamicSerDeStart.java */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+public class DynamicSerDeStart extends SimpleNode {
+  public DynamicSerDeStart(int id) {
+    super(id);
+  }
+
+  public DynamicSerDeStart(thrift_grammar p, int id) {
+    super(p, id);
+  }
+
+}

Modified: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeStructBase.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeStructBase.java?rev=712905&r1=712904&r2=712905&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeStructBase.java (original)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeStructBase.java Mon Nov 10 17:50:06 2008
@@ -81,7 +81,6 @@
     fieldList.serialize(o, oi, oprot);
 
     if(thrift_mode) {
-      oprot.writeFieldStop();
       oprot.writeStructEnd();
     }
   }

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeThrows.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeThrows.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeThrows.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeThrows.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,14 @@
+/* Generated By:JJTree: Do not edit this line. DynamicSerDeThrows.java */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+public class DynamicSerDeThrows extends SimpleNode {
+  public DynamicSerDeThrows(int id) {
+    super(id);
+  }
+
+  public DynamicSerDeThrows(thrift_grammar p, int id) {
+    super(p, id);
+  }
+
+}