svn commit: r1452992 [1/8] - in /hive/trunk: ./ ivy/ ql/ ql/src/gen/protobuf/ ql/src/gen/protobuf/gen-java/ ql/src/gen/protobuf/gen-java/org/ ql/src/gen/protobuf/gen-java/org/apache/ ql/src/gen/protobuf/gen-java/org/apache/hadoop/ ql/src/gen/protobuf/g...

[ke...@apache.org]

Author: kevinwilfong
Date: Tue Mar  5 20:44:50 2013
New Revision: 1452992

URL: http://svn.apache.org/r1452992
Log:
HIVE-3874. Create a new Optimized Row Columnar file format for Hive. (Owen O'Malley via kevinwilfong)

Added:
    hive/trunk/ql/src/gen/protobuf/
    hive/trunk/ql/src/gen/protobuf/gen-java/
    hive/trunk/ql/src/gen/protobuf/gen-java/org/
    hive/trunk/ql/src/gen/protobuf/gen-java/org/apache/
    hive/trunk/ql/src/gen/protobuf/gen-java/org/apache/hadoop/
    hive/trunk/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/
    hive/trunk/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/
    hive/trunk/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/
    hive/trunk/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/
    hive/trunk/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/BitFieldReader.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/BitFieldWriter.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/BooleanColumnStatistics.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatistics.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/CompressionCodec.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/CompressionKind.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DoubleColumnStatistics.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DynamicByteArray.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/DynamicIntArray.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/InStream.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/IntegerColumnStatistics.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUnion.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OutStream.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/PositionProvider.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/PositionRecorder.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/PositionedOutputStream.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReader.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RedBlackTree.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthByteReader.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthByteWriter.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerReader.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerWriter.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SerializationUtils.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SnappyCodec.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StreamName.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StringColumnStatistics.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StringRedBlackTree.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StripeInformation.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Writer.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ZlibCodec.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/package-info.java
    hive/trunk/ql/src/protobuf/
    hive/trunk/ql/src/protobuf/org/
    hive/trunk/ql/src/protobuf/org/apache/
    hive/trunk/ql/src/protobuf/org/apache/hadoop/
    hive/trunk/ql/src/protobuf/org/apache/hadoop/hive/
    hive/trunk/ql/src/protobuf/org/apache/hadoop/hive/ql/
    hive/trunk/ql/src/protobuf/org/apache/hadoop/hive/ql/io/
    hive/trunk/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/
    hive/trunk/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto
    hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/
    hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestBitFieldReader.java
    hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestDynamicArray.java
    hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java
    hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInStream.java
    hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
    hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
    hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcStruct.java
    hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRunLengthByteReader.java
    hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRunLengthIntegerReader.java
    hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestSerializationUtils.java
    hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStreamName.java
    hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStringRedBlackTree.java
    hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestZlib.java
    hive/trunk/ql/src/test/resources/orc-file-dump.out
Modified:
    hive/trunk/build.properties
    hive/trunk/build.xml
    hive/trunk/ivy/libraries.properties
    hive/trunk/ql/build.xml
    hive/trunk/ql/ivy.xml

Modified: hive/trunk/build.properties
URL: http://svn.apache.org/viewvc/hive/trunk/build.properties?rev=1452992&r1=1452991&r2=1452992&view=diff
==============================================================================
--- hive/trunk/build.properties (original)
+++ hive/trunk/build.properties Tue Mar  5 20:44:50 2013
@@ -76,6 +76,7 @@ iterate.hive.all=ant,shims,common,serde,
 iterate.hive.modules=shims,common,serde,metastore,ql,contrib,service,cli,jdbc,hwi,hbase-handler,pdk,builtins
 iterate.hive.tests=ql,contrib,hbase-handler,hwi,jdbc,metastore,odbc,serde,service
 iterate.hive.thrift=ql,service,metastore,serde
+iterate.hive.protobuf=ql
 iterate.hive.cpp=odbc
 
 #

Modified: hive/trunk/build.xml
URL: http://svn.apache.org/viewvc/hive/trunk/build.xml?rev=1452992&r1=1452991&r2=1452992&view=diff
==============================================================================
--- hive/trunk/build.xml (original)
+++ hive/trunk/build.xml Tue Mar  5 20:44:50 2013
@@ -256,6 +256,11 @@
     <iterate target="thriftif" iterate="${iterate.hive.thrift}"/>
   </target>
 
+  <target name="protobuf">
+    <echo message="Project: ${ant.project.name}"/>
+    <iterate target="protobuf" iterate="${iterate.hive.protobuf}"/>
+  </target>
+
   <target name="jar"
           depends="init"
           description="Build JAR artifacts">

Modified: hive/trunk/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/hive/trunk/ivy/libraries.properties?rev=1452992&r1=1452991&r2=1452992&view=diff
==============================================================================
--- hive/trunk/ivy/libraries.properties (original)
+++ hive/trunk/ivy/libraries.properties Tue Mar  5 20:44:50 2013
@@ -60,9 +60,11 @@ libthrift.version=0.9.0
 log4j.version=1.2.16
 maven-ant-tasks.version=2.1.0
 mockito-all.version=1.8.2
+protobuf.version=2.4.1
 rat.version=0.8
 slf4j-api.version=1.6.1
 slf4j-log4j12.version=1.6.1
+snappy.version=0.2
 velocity.version=1.5
 zookeeper.version=3.4.3
 javolution.version=5.5.1

Modified: hive/trunk/ql/build.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ql/build.xml?rev=1452992&r1=1452991&r2=1452992&view=diff
==============================================================================
--- hive/trunk/ql/build.xml (original)
+++ hive/trunk/ql/build.xml Tue Mar  5 20:44:50 2013
@@ -22,6 +22,9 @@
 
   <property name="ql.lib.dir" value="${basedir}/lib"/>
   <property name="src.dir"  location="${basedir}/src/java"/>
+  <property name="protobuf.src.dir"  location="${basedir}/src/protobuf"/>
+  <property name="protobuf.build.dir"  
+            location="${basedir}/src/gen/protobuf/gen-java"/>
   <property name="ql.test.query.dir" location="${basedir}/src/test/queries"/>
   <property name="ql.test.template.dir" location="${basedir}/src/test/templates"/>
   <property name="ql.test.results.dir" location="${basedir}/src/test/results"/>
@@ -147,16 +150,28 @@
     </java>
   </target>
 
+  <target name="protobuf">
+    <echo message="Project: ${ant.project.name}"/>
+    <echo>Building ORC Protobuf</echo>
+    <mkdir dir="${protobuf.build.dir}"/>
+    <exec executable="protoc" failonerror="true">
+      <arg value="--java_out=${protobuf.build.dir}"/>
+      <arg value="-I=${protobuf.src.dir}/org/apache/hadoop/hive/ql/io/orc"/>
+      <arg value="${protobuf.src.dir}/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto"/>
+    </exec>
+  </target>
+
   <target name="ql-init">
     <echo message="Project: ${ant.project.name}"/>
     <mkdir dir="${build.dir}/gen/antlr/gen-java/org/apache/hadoop/hive/ql/parse"/>
   </target>
 
-  <target name="compile" depends="init, ql-init, ivy-retrieve, build-grammar">
+  <target name="compile" 
+          depends="init, ql-init, ivy-retrieve, build-grammar">
     <echo message="Project: ${ant.project.name}"/>
     <javac
      encoding="${build.encoding}"
-     srcdir="${src.dir}:${basedir}/src/gen/thrift/gen-javabean:${build.dir}/gen/antlr/gen-java"
+     srcdir="${src.dir}:${basedir}/src/gen/thrift/gen-javabean:${build.dir}/gen/antlr/gen-java:${protobuf.build.dir}"
      includes="**/*.java"
      destdir="${build.classes}"
      debug="${javac.debug}"

Modified: hive/trunk/ql/ivy.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ql/ivy.xml?rev=1452992&r1=1452991&r2=1452992&view=diff
==============================================================================
--- hive/trunk/ql/ivy.xml (original)
+++ hive/trunk/ql/ivy.xml Tue Mar  5 20:44:50 2013
@@ -38,6 +38,10 @@
                 conf="test->default" transitive="false"/>
     <dependency org="org.antlr" name="antlr" rev="${antlr.version}"
                 transitive="false"/>
+    <dependency org="com.google.protobuf" name="protobuf-java" 
+                rev="${protobuf.version}" transitive="false"/>
+    <dependency org="org.iq80.snappy" name="snappy" 
+                rev="${snappy.version}" transitive="false"/>
 
     <!-- hadoop specific guava -->
     <dependency org="com.google.guava" name="guava" rev="${guava-hadoop20.version}"