You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2014/07/29 07:06:10 UTC

svn commit: r1614252 - in /pig/trunk: ./ src/org/apache/pig/builtin/ src/org/apache/pig/impl/util/orc/ test/e2e/pig/ test/e2e/pig/conf/ test/e2e/pig/drivers/ test/e2e/pig/tests/ test/org/apache/pig/builtin/

Author: rohini
Date: Tue Jul 29 05:06:09 2014
New Revision: 1614252

URL: http://svn.apache.org/r1614252
Log:
PIG-4077: Some fixes and e2e test for OrcStorage (rohini)

Added:
    pig/trunk/test/e2e/pig/tests/orc.conf
Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/builtin/OrcStorage.java
    pig/trunk/src/org/apache/pig/impl/util/orc/OrcUtils.java
    pig/trunk/test/e2e/pig/build.xml
    pig/trunk/test/e2e/pig/conf/default.conf
    pig/trunk/test/e2e/pig/conf/local.conf
    pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm
    pig/trunk/test/org/apache/pig/builtin/TestOrcStorage.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1614252&r1=1614251&r2=1614252&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Jul 29 05:06:09 2014
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
  
 IMPROVEMENTS
 
+PIG-4077: Some fixes and e2e test for OrcStorage (rohini)
+
 PIG-4054: Do not create job.jar when submitting job (daijy)
 
 PIG-4047: Break up pig withouthadoop and fat jar (daijy)

Modified: pig/trunk/src/org/apache/pig/builtin/OrcStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/builtin/OrcStorage.java?rev=1614252&r1=1614251&r2=1614252&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/builtin/OrcStorage.java (original)
+++ pig/trunk/src/org/apache/pig/builtin/OrcStorage.java Tue Jul 29 05:06:09 2014
@@ -83,9 +83,9 @@ import org.apache.pig.impl.util.orc.OrcU
  * <ul>
  * <li><code>-s, --stripeSize</code> Set the stripe size for the file
  * <li><code>-r, --rowIndexStride</code> Set the distance between entries in the row index
- * <li><code>-b, --bufferSize</code> The size of the memory buffers used for compressing and storing the 
+ * <li><code>-b, --bufferSize</code> The size of the memory buffers used for compressing and storing the
  * stripe in memory
- * <li><code>-p, --blockPadding</code> Sets whether the HDFS blocks are padded to prevent stripes 
+ * <li><code>-p, --blockPadding</code> Sets whether the HDFS blocks are padded to prevent stripes
  * from straddling blocks
  * <li><code>-c, --compress</code> Sets the generic compression that is used to compress the data.
  * Valid codecs are: NONE, ZLIB, SNAPPY, LZO
@@ -105,16 +105,16 @@ public class OrcStorage extends LoadFunc
     private Integer bufferSize;
     private Boolean blockPadding;
     private CompressionKind compress;
-    private org.apache.hadoop.hive.ql.io.orc.OrcFile.Version version;
-    
+    private Version version;
+
     private static final Options validOptions;
     private final CommandLineParser parser = new GnuParser();
-    protected final Log log = LogFactory.getLog(getClass());
+    protected final static Log log = LogFactory.getLog(OrcStorage.class);
     protected boolean[] mRequiredColumns = null;
-    
+
     private static final String SchemaSignatureSuffix = "_schema";
     private static final String RequiredColumnsSuffix = "_columns";
-    
+
     static {
         validOptions = new Options();
         validOptions.addOption("s", "stripeSize", true,
@@ -131,10 +131,10 @@ public class OrcStorage extends LoadFunc
         validOptions.addOption("v", "version", true,
                 "Sets the version of the file that will be written");
     }
-    
+
     public OrcStorage() {
     }
-    
+
     public OrcStorage(String options) {
         String[] optsArr = options.split(" ");
         try {
@@ -200,7 +200,7 @@ public class OrcStorage extends LoadFunc
             }
             if (version!=null) {
                 job.getConfiguration().set(HiveConf.ConfVars.HIVE_ORC_WRITE_FORMAT.varname,
-                        version.toString());
+                        version.getName());
             }
         }
         FileOutputFormat.setOutputPath(job, new Path(location));
@@ -241,7 +241,7 @@ public class OrcStorage extends LoadFunc
     public void setStoreFuncUDFContextSignature(String signature) {
         this.signature = signature;
     }
-    
+
     @Override
     public void setUDFContextSignature(String signature) {
         this.signature = signature;
@@ -261,20 +261,22 @@ public class OrcStorage extends LoadFunc
         Properties p = UDFContext.getUDFContext().getUDFProperties(this.getClass());
         if (!UDFContext.getUDFContext().isFrontend()) {
             typeInfo = (TypeInfo)ObjectSerializer.deserialize(p.getProperty(signature + SchemaSignatureSuffix));
+        } else if (typeInfo == null) {
+            typeInfo = getTypeInfo(location, job);
         }
-        if (typeInfo!=null && oi==null) {
+        if (typeInfo != null && oi == null) {
             oi = OrcStruct.createObjectInspector(typeInfo);
         }
         if (!UDFContext.getUDFContext().isFrontend() &&
                 p.getProperty(signature+RequiredColumnsSuffix)!=null) {
             mRequiredColumns = (boolean[])ObjectSerializer.deserialize(p.getProperty(signature+RequiredColumnsSuffix));
             job.getConfiguration().setBoolean(ColumnProjectionUtils.READ_ALL_COLUMNS, false);
-            job.getConfiguration().set(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR, 
+            job.getConfiguration().set(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR,
                     getReqiredColumnIdString(mRequiredColumns));
         }
         FileInputFormat.setInputPaths(job, location);
     }
-    
+
     private static String getReqiredColumnIdString(boolean[] requiredColumns) {
         String result = "";
         for (int i=0;i<requiredColumns.length;i++) {
@@ -308,7 +310,7 @@ public class OrcStorage extends LoadFunc
                 return null;
             }
             Object value = in.getCurrentValue();
-            
+
             Tuple t = (Tuple)OrcUtils.convertOrcToPig(value, oi, mRequiredColumns);
             return t;
         } catch (InterruptedException e) {
@@ -318,7 +320,7 @@ public class OrcStorage extends LoadFunc
                     PigException.REMOTE_ENVIRONMENT, e);
         }
     }
-    
+
     private static Path getFirstFile(String location, FileSystem fs) throws IOException {
         String[] locations = getPathStrings(location);
         Path[] paths = new Path[locations.length];
@@ -327,8 +329,11 @@ public class OrcStorage extends LoadFunc
         }
         List<FileStatus> statusList = new ArrayList<FileStatus>();
         for (int i = 0; i < paths.length; ++i) {
-            for (FileStatus tempf : fs.globStatus(paths[i])) {
-                statusList.add(tempf);
+            FileStatus[] files = fs.globStatus(paths[i]);
+            if (files != null) {
+                for (FileStatus tempf : files) {
+                    statusList.add(tempf);
+                }
             }
         }
         FileStatus[] statusArray = (FileStatus[]) statusList
@@ -340,26 +345,46 @@ public class OrcStorage extends LoadFunc
     private static TypeInfo getTypeInfoFromLocation(String location, Job job) throws IOException {
         FileSystem fs = FileSystem.get(job.getConfiguration());
         Path path = getFirstFile(location, fs);
-        if (path==null) {
-            throw new IOException("Cannot find any ORC files from " + location);
+        if (path == null) {
+            log.info("Cannot find any ORC files from " + location +
+                    ". Probably multiple load store in script.");
+            return null;
         }
         Reader reader = OrcFile.createReader(fs, path);
         ObjectInspector oip = (ObjectInspector)reader.getObjectInspector();
         return TypeInfoUtils.getTypeInfoFromObjectInspector(oip);
     }
-    
+
     @Override
     public ResourceSchema getSchema(String location, Job job)
             throws IOException {
-        if (typeInfo==null) {
-            typeInfo = getTypeInfoFromLocation(location, job);
-            Properties p = UDFContext.getUDFContext().getUDFProperties(this.getClass());
-            p.setProperty(signature + SchemaSignatureSuffix, ObjectSerializer.serialize(typeInfo));
+        if (typeInfo == null) {
+            typeInfo = getTypeInfo(location, job);
+            // still null means case of multiple load store
+            if (typeInfo == null) {
+                return null;
+            }
         }
+
         ResourceFieldSchema fs = OrcUtils.getResourceFieldSchema(typeInfo);
         return fs.getSchema();
     }
 
+    private TypeInfo getTypeInfo(String location, Job job) throws IOException {
+        Properties p = UDFContext.getUDFContext().getUDFProperties(this.getClass());
+        String serializedStr = p.getProperty(signature + SchemaSignatureSuffix);
+        TypeInfo typeInfo = null;
+        if (serializedStr != null) {
+            typeInfo = (TypeInfo) ObjectSerializer.deserialize(serializedStr);
+        } else {
+            typeInfo = getTypeInfoFromLocation(location, job);
+        }
+        if (typeInfo != null) {
+            p.setProperty(signature + SchemaSignatureSuffix, ObjectSerializer.serialize(typeInfo));
+        }
+        return typeInfo;
+    }
+
     @Override
     public ResourceStatistics getStatistics(String location, Job job)
             throws IOException {

Modified: pig/trunk/src/org/apache/pig/impl/util/orc/OrcUtils.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/util/orc/OrcUtils.java?rev=1614252&r1=1614251&r2=1614252&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/util/orc/OrcUtils.java (original)
+++ pig/trunk/src/org/apache/pig/impl/util/orc/OrcUtils.java Tue Jul 29 05:06:09 2014
@@ -20,6 +20,7 @@ package org.apache.pig.impl.util.orc;
 import java.io.IOException;
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import java.sql.Timestamp;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -27,8 +28,8 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.hadoop.hive.common.type.HiveDecimal;
-import org.apache.hadoop.hive.serde2.io.DateWritable;
 import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
+import org.apache.hadoop.hive.serde2.io.TimestampWritable;
 import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -36,16 +37,10 @@ import org.apache.hadoop.hive.serde2.obj
 import org.apache.hadoop.hive.serde2.objectinspector.StructField;
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableBooleanObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableDoubleObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableFloatObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableIntObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableLongObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableStringObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableTimestampObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector;
 import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
@@ -66,7 +61,10 @@ import org.joda.time.DateTime;
 
 public class OrcUtils {
     public static Object convertOrcToPig(Object obj, ObjectInspector oi, boolean[] includedColumns) {
-        Object result;
+        Object result = null;
+        if (obj == null) {
+            return result;
+        }
         switch (oi.getCategory()) {
         case PRIMITIVE:
             PrimitiveObjectInspector poi = (PrimitiveObjectInspector)oi;
@@ -112,9 +110,12 @@ public class OrcUtils {
         }
         return result;
     }
-    
+
     public static Object getPrimaryFromOrc(Object obj, PrimitiveObjectInspector poi) {
-        Object result;
+        Object result = null;
+        if (obj == null) {
+            return result;
+        }
         switch (poi.getPrimitiveCategory()) {
         case FLOAT:
         case DOUBLE:
@@ -131,9 +132,9 @@ public class OrcUtils {
             result = (int)(Short)poi.getPrimitiveJavaObject(obj);
             break;
         case BINARY:
-            byte[] r = new byte[((BytesWritable)obj).getLength()];
-            System.arraycopy(((BytesWritable)obj).getBytes(), 0, r, 0, ((BytesWritable)obj).getLength());
-            result = new DataByteArray(r);
+            BytesWritable bw = (BytesWritable) obj;
+            // Make a copy
+            result = new DataByteArray(bw.getBytes(), 0, bw.getLength());
             break;
         case TIMESTAMP:
             java.sql.Timestamp origTimeStamp = (java.sql.Timestamp)poi.getPrimitiveJavaObject(obj);
@@ -144,7 +145,7 @@ public class OrcUtils {
             result = new DateTime(origDate.getTime());
             break;
         case DECIMAL:
-            org.apache.hadoop.hive.common.type.HiveDecimal origDecimal = 
+            org.apache.hadoop.hive.common.type.HiveDecimal origDecimal =
                 (org.apache.hadoop.hive.common.type.HiveDecimal)poi.getPrimitiveJavaObject(obj);
             result = origDecimal.bigDecimalValue();
             break;
@@ -154,50 +155,7 @@ public class OrcUtils {
         }
         return result;
     }
-    
-    public static Object getPrimaryFromPig(Object obj, PrimitiveObjectInspector poi) throws IOException {
-        Object result=null;
-        switch (poi.getPrimitiveCategory()) {
-        case FLOAT:
-            result = ((SettableFloatObjectInspector)poi).create((Float)obj);
-            break;
-        case DOUBLE:
-            result = ((SettableDoubleObjectInspector)poi).create((Double)obj);
-            break;
-        case BOOLEAN:
-            result = ((SettableBooleanObjectInspector)poi).create((Boolean)obj);
-            break;
-        case INT:
-            result = ((SettableIntObjectInspector)poi).create((Integer)obj);
-            break;
-        case LONG:
-            result = ((SettableLongObjectInspector)poi).create((Long)obj);
-            break;
-        case STRING:
-            result = ((SettableStringObjectInspector)poi).create((String)obj);
-            break;
-        case TIMESTAMP:
-            DateTime origDateTime = (DateTime)obj;
-            result = ((SettableTimestampObjectInspector)poi).create(new java.sql.Timestamp(origDateTime.getMillis()));
-            break;
-        case DECIMAL:
-            if (obj instanceof BigDecimal) {
-                result = org.apache.hadoop.hive.common.type.HiveDecimal.create((BigDecimal)obj);
-            } else if (obj instanceof BigInteger) {
-                result = org.apache.hadoop.hive.common.type.HiveDecimal.create((BigInteger)obj);
-            } else {
-                throw new IOException("Should never happen, except BigDecimal/BigInteger, but get " + obj.getClass());
-            }
-            break;
-        case DATE:
-        case BYTE:
-        case SHORT:
-        case BINARY:
-            throw new IOException("Should never happen, expect a binary, which is not valid Pig data type");
-        }
-        return result;
-    }
-    
+
     public static ResourceFieldSchema getResourceFieldSchema(TypeInfo ti) throws IOException {
         ResourceFieldSchema fieldSchema = new ResourceFieldSchema();
         ResourceFieldSchema[] innerFs;
@@ -279,10 +237,10 @@ public class OrcUtils {
             }
             break;
         }
-        
+
         return fieldSchema;
     }
-    
+
     public static TypeInfo getTypeInfo(ResourceFieldSchema fs) throws IOException {
         TypeInfo ti;
         switch (fs.getType()) {
@@ -308,10 +266,12 @@ public class OrcUtils {
             break;
         case DataType.MAP:
             ti = new MapTypeInfo();
-            if (fs.getSchema()==null || fs.getSchema().getFields().length!=1) {
-                throw new IOException("Wrong map inner schema");
+            TypeInfo valueField;
+            if (fs.getSchema() == null || fs.getSchema().getFields().length != 1) {
+                valueField = TypeInfoFactory.binaryTypeInfo;
+            } else {
+                valueField = getTypeInfo(fs.getSchema().getFields()[0]);
             }
-            TypeInfo valueField = getTypeInfo(fs.getSchema().getFields()[0]);
             ((MapTypeInfo)ti).setMapKeyTypeInfo(TypeInfoFactory.stringTypeInfo);
             ((MapTypeInfo)ti).setMapValueTypeInfo(valueField);
             break;
@@ -342,13 +302,16 @@ public class OrcUtils {
         case DataType.BIGINTEGER:
             ti = TypeInfoFactory.decimalTypeInfo;
             break;
+        case DataType.BYTEARRAY:
+            ti = TypeInfoFactory.binaryTypeInfo;
+            break;
         default:
             throw new IllegalArgumentException("Unknown data type " +
                 DataType.findTypeName(fs.getType()));
         }
         return ti;
     }
-    
+
     static class Field implements StructField {
         private final String name;
         private final ObjectInspector inspector;
@@ -380,7 +343,7 @@ public class OrcUtils {
           return null;
         }
       }
-    
+
     static class PigStructInspector extends StructObjectInspector {
         private List<StructField> fields;
 
@@ -471,7 +434,7 @@ public class OrcUtils {
             }
         }
     }
-    
+
     static class PigMapObjectInspector implements MapObjectInspector {
         private ObjectInspector key;
         private ObjectInspector value;
@@ -480,7 +443,7 @@ public class OrcUtils {
             key = createObjectInspector(info.getMapKeyTypeInfo());
             value = createObjectInspector(info.getMapValueTypeInfo());
         }
-        
+
         @Override
         public ObjectInspector getMapKeyObjectInspector() {
             return key;
@@ -528,7 +491,7 @@ public class OrcUtils {
             }
         }
     }
-    
+
     static class PigListObjectInspector implements ListObjectInspector {
         private ObjectInspector child;
         private Object cachedObject;
@@ -598,25 +561,46 @@ public class OrcUtils {
             }
         }
     }
-    
-    static class JodaDateObjectInspector extends
-            AbstractPrimitiveJavaObjectInspector implements DateObjectInspector {
 
-        protected JodaDateObjectInspector() {
-            super(TypeInfoFactory.dateTypeInfo);
+    static class PigDataByteArrayObjectInspector extends AbstractPrimitiveJavaObjectInspector
+            implements BinaryObjectInspector {
+
+        PigDataByteArrayObjectInspector() {
+            super(TypeInfoFactory.binaryTypeInfo);
         }
 
         @Override
-        public DateWritable getPrimitiveWritableObject(Object o) {
-            return o == null ? null : new DateWritable(new java.sql.Date(((DateTime)o).getMillis()));
+        public BytesWritable getPrimitiveWritableObject(Object o) {
+            return o == null ? null : (o instanceof DataByteArray
+                            ? new BytesWritable(((DataByteArray) o).get())
+                            : new BytesWritable((byte[]) o));
         }
 
         @Override
-        public java.sql.Date getPrimitiveJavaObject(Object o) {
-            return o == null ? null : new java.sql.Date(((DateTime)o).getMillis());
+        public byte[] getPrimitiveJavaObject(Object o) {
+            return ((DataByteArray) o).get();
         }
+
     }
-    
+
+    static class PigJodaTimeStampObjectInspector extends
+            AbstractPrimitiveJavaObjectInspector implements TimestampObjectInspector {
+
+        protected PigJodaTimeStampObjectInspector() {
+            super(TypeInfoFactory.timestampTypeInfo);
+        }
+
+        @Override
+        public TimestampWritable getPrimitiveWritableObject(Object o) {
+            return o == null ? null : new TimestampWritable(new Timestamp(((DateTime)o).getMillis()));
+        }
+
+        @Override
+        public Timestamp getPrimitiveJavaObject(Object o) {
+            return o == null ? null : new Timestamp(((DateTime)o).getMillis());
+        }
+    }
+
     static class PigDecimalObjectInspector extends
             AbstractPrimitiveJavaObjectInspector implements HiveDecimalObjectInspector {
 
@@ -642,7 +626,7 @@ public class OrcUtils {
             }
         }
     }
-    
+
     public static ObjectInspector createObjectInspector(TypeInfo info) {
         switch (info.getCategory()) {
         case PRIMITIVE:
@@ -660,14 +644,15 @@ public class OrcUtils {
             case STRING:
               return PrimitiveObjectInspectorFactory.javaStringObjectInspector;
             case TIMESTAMP:
-              return new JodaDateObjectInspector();
+              return new PigJodaTimeStampObjectInspector();
             case DECIMAL:
               return new PigDecimalObjectInspector();
+            case BINARY:
+              return new PigDataByteArrayObjectInspector();
             case DATE:
             case VARCHAR:
             case BYTE:
             case SHORT:
-            case BINARY:
                 throw new IllegalArgumentException("Should never happen, " + 
                         (((PrimitiveTypeInfo) info).getPrimitiveCategory()) +
                         "is not valid Pig primitive data type");

Modified: pig/trunk/test/e2e/pig/build.xml
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/build.xml?rev=1614252&r1=1614251&r2=1614252&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/build.xml (original)
+++ pig/trunk/test/e2e/pig/build.xml Tue Jul 29 05:06:09 2014
@@ -25,6 +25,12 @@
     value="${pig.dir}/build/ivy/lib/Pig/jython-standalone-${jython.version}.jar"/>
   <property name="jruby.jar"
     value="${pig.dir}/build/ivy/lib/Pig/jruby-complete-${jruby.version}.jar"/>
+  <property name="hive.lib.dir"
+	value="${pig.dir}/build/ivy/lib/Pig"/>
+
+  <condition property="hive.hadoop.shims.version" value="0.23" else="0.20S">
+    <equals arg1="${hadoopversion}" arg2="23" />
+  </condition>
 
   <!-- Separate property name for udfs' build.xml -->
   <property name="udf.dir" value="${basedir}/udfs"/>
@@ -275,6 +281,9 @@
       <env key="PH_CLUSTER_BIN" value="${harness.cluster.bin}"/>
       <env key="PH_JYTHON_JAR" value="${jython.jar}"/>
       <env key="PH_JRUBY_JAR" value="${jruby.jar}"/>
+      <env key="PH_HIVE_LIB_DIR" value="${hive.lib.dir}"/>
+      <env key="PH_HIVE_VERSION" value="${hive.version}"/>
+      <env key="PH_HIVE_SHIMS_VERSION" value="${hive.hadoop.shims.version}"/>
       <env key="JRUBY_OPTS" value="--1.9"/>
       <env key="HARNESS_CONF" value="${harness.conf.file}"/>
       <env key="HADOOP_HOME" value="${harness.hadoop.home}"/>
@@ -306,6 +315,7 @@
       <arg value="${test.location}/tests/bigdata.conf"/>
       <arg value="${test.location}/tests/grunt.conf"/>
       <arg value="${test.location}/tests/macro.conf"/>
+      <arg value="${test.location}/tests/orc.conf"/>
       <arg value="${test.location}/tests/hcat.conf"/>
     </exec>
   </target>

Modified: pig/trunk/test/e2e/pig/conf/default.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/conf/default.conf?rev=1614252&r1=1614251&r2=1614252&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/conf/default.conf (original)
+++ pig/trunk/test/e2e/pig/conf/default.conf Tue Jul 29 05:06:09 2014
@@ -61,6 +61,10 @@ $cfg = {
     #HADOOP
     , 'mapredjars'       => "$ENV{PH_ROOT}/lib"
 
+    #HIVE
+    , 'hivelibdir'       => "$ENV{PH_HIVE_LIB_DIR}"
+    , 'hiveversion'      =>  "$ENV{PH_HIVE_VERSION}"
+    , 'hiveshimsversion' => "$ENV{PH_HIVE_SHIMS_VERSION}"
 
     , 'userhomePath' => "$ENV{HOME}"
     ,'local.bin'     => '/usr/bin'

Modified: pig/trunk/test/e2e/pig/conf/local.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/conf/local.conf?rev=1614252&r1=1614251&r2=1614252&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/conf/local.conf (original)
+++ pig/trunk/test/e2e/pig/conf/local.conf Tue Jul 29 05:06:09 2014
@@ -57,6 +57,11 @@ $cfg = {
 	, 'mapredjars'       => "$ENV{PH_ROOT}/lib"
 	, 'hadoopHome'       => "$ENV{PH_ROOT}/lib"
 
+	#HIVE
+    , 'hivelibdir'       => "$ENV{PH_HIVE_LIB_DIR}"
+    , 'hiveversion'      =>  "$ENV{PH_HIVE_VERSION}"
+    , 'hiveshimsversion' => "$ENV{PH_HIVE_SHIMS_VERSION}"
+
     , 'userhomePath' => "$ENV{HOME}"
     ,'local.bin'     => '/usr/bin'
     # NB: we provide the default there as a fallback in case if there is no corresponding ENV key:

Modified: pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm?rev=1614252&r1=1614251&r2=1614252&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm (original)
+++ pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm Tue Jul 29 05:06:09 2014
@@ -78,6 +78,9 @@ sub replaceParameters
     $cmd =~ s/:RUNID:/$testCmd->{'UID'}/g;
     $cmd =~ s/:USRHOMEPATH:/$testCmd->{'userhomePath'}/g;
     $cmd =~ s/:MAPREDJARS:/$testCmd->{'mapredjars'}/g;
+    $cmd =~ s/:HIVELIBDIR:/$testCmd->{'hivelibdir'}/g;
+    $cmd =~ s/:HIVEVERSION:/$testCmd->{'hiveversion'}/g;
+    $cmd =~ s/:HIVESHIMSVERSION:/$testCmd->{'hiveshimsversion'}/g;
     $cmd =~ s/:SCRIPTHOMEPATH:/$testCmd->{'scriptPath'}/g;
     $cmd =~ s/:DBUSER:/$testCmd->{'dbuser'}/g;
     $cmd =~ s/:DBNAME:/$testCmd->{'dbdb'}/g;

Added: pig/trunk/test/e2e/pig/tests/orc.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/tests/orc.conf?rev=1614252&view=auto
==============================================================================
--- pig/trunk/test/e2e/pig/tests/orc.conf (added)
+++ pig/trunk/test/e2e/pig/tests/orc.conf Tue Jul 29 05:06:09 2014
@@ -0,0 +1,109 @@
+$cfg = {
+        'driver' => 'Pig',
+        'nummachines' => 5,
+        'verify_with_pig' => 1,
+        'verify_pig_version' => 'old',
+
+        'groups' => [
+                {
+                'name' => 'Orc',
+                'tests' => [
+# Test 1: Load (primitive) from PigStorage and store into OrcStorage
+# Also tests multiple load stores in same script
+                        {
+                        'num' => 1,
+                        'notmq' => 1,
+                        'pig' => q\
+register :HIVELIBDIR:/hive-common-:HIVEVERSION:.jar;
+register :HIVELIBDIR:/hive-serde-:HIVEVERSION:.jar;
+register :HIVELIBDIR:/hive-exec-:HIVEVERSION:-core.jar;
+register :HIVELIBDIR:/hive-shims-common-:HIVEVERSION:.jar;
+register :HIVELIBDIR:/hive-shims-common-secure-:HIVEVERSION:.jar;
+register :HIVELIBDIR:/hive-shims-:HIVESHIMSVERSION:-:HIVEVERSION:.jar;
+a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:float);
+store a into ':OUTPATH:.intermediate' using OrcStorage();
+b = load ':OUTPATH:.intermediate' using OrcStorage();
+describe b;
+c = filter b by age < 30;
+store c into ':OUTPATH:';\,
+                        'verify_pig_script' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:float);
+b = filter a by age < 30;
+store b into ':OUTPATH:';\,
+                        },
+# Test 2: Load (complex) from PigStorage and store into OrcStorage
+                        {
+                        'num' => 2,
+                        'notmq' => 1,
+                        'pig' => q\
+register :HIVELIBDIR:/hive-common-:HIVEVERSION:.jar;
+register :HIVELIBDIR:/hive-serde-:HIVEVERSION:.jar;
+register :HIVELIBDIR:/hive-exec-:HIVEVERSION:-core.jar;
+register :HIVELIBDIR:/hive-shims-common-:HIVEVERSION:.jar;
+register :HIVELIBDIR:/hive-shims-common-secure-:HIVEVERSION:.jar;
+register :HIVELIBDIR:/hive-shims-:HIVESHIMSVERSION:-:HIVEVERSION:.jar;
+a = load ':INPATH:/singlefile/studentcomplextab10k' as (nameagegpamap:map[], nameagegpatuple:tuple(tname:chararray, tage:int, tgpa:float), nameagegpabag:bag{t:tuple(bname:chararray, bage:int, bgpa:float)});
+store a into ':OUTPATH:.intermediate' using OrcStorage();
+b = load ':OUTPATH:.intermediate' using OrcStorage();
+store b into ':OUTPATH:';\,
+                        'verify_pig_script' => q\a = load ':INPATH:/singlefile/studentcomplextab10k' as (nameagegpamap:map[], nameagegpatuple:tuple(tname:chararray, tage:int, tgpa:float), nameagegpabag:bag{t:tuple(bname:chararray, bage:int, bgpa:float)});
+store a into ':OUTPATH:';\,
+                        },
+# Test 3: Aggregation test using two ORCStorage datasets
+# Also incorporates handling bytearrays
+                        {
+                        'num' => 3,
+                        'notmq' => 1,
+                        'pig' => q\register :HIVELIBDIR:/hive-common-:HIVEVERSION:.jar;
+register :HIVELIBDIR:/hive-serde-:HIVEVERSION:.jar;
+register :HIVELIBDIR:/hive-exec-:HIVEVERSION:-core.jar;
+register :HIVELIBDIR:/hive-shims-common-:HIVEVERSION:.jar;
+register :HIVELIBDIR:/hive-shims-common-secure-:HIVEVERSION:.jar;
+register :HIVELIBDIR:/hive-shims-:HIVESHIMSVERSION:-:HIVEVERSION:.jar;
+a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:float);
+store a into ':OUTPATH:.simple.intermediate' using OrcStorage();
+b = load ':INPATH:/singlefile/studentcomplextab10k' as (nameagegpamap:map[], nameagegpatuple:tuple(tname:chararray, tage:int, tgpa:float), nameagegpabag:bag{t:tuple(bname:chararray, bage:int, bgpa:float)}, nameagegpamap_name:chararray, nameagegpamap_age:int, nameagegpamap_gpa:float);
+store b into ':OUTPATH:.complex.intermediate' using OrcStorage();
+c = load ':OUTPATH:.simple.intermediate' using OrcStorage();
+describe c;
+d = load ':OUTPATH:.complex.intermediate' using OrcStorage();
+describe d;
+e = foreach c generate name, age, gpa;
+f = foreach d generate nameagegpamap#'name' as name, nameagegpamap#'age' as age, nameagegpamap#'gpa' as gpa, nameagegpatuple.tage as tage, FLATTEN(nameagegpabag) as (bname, bage, bgpa);
+g = join e by name, f by name;
+h = group g by (f::bgpa);
+j = foreach h generate group, COUNT(g) as students;
+k = order j by group, students;
+store k into ':OUTPATH:';\,
+                        'verify_pig_script' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:float);
+b = load ':INPATH:/singlefile/studentcomplextab10k' as (nameagegpamap:map[], nameagegpatuple:tuple(tname:chararray, tage:int, tgpa:float), nameagegpabag:bag{t:tuple(bname:chararray, bage:int, bgpa:float)}, nameagegpamap_name:chararray, nameagegpamap_age:int, nameagegpamap_gpa:float);
+c = foreach a generate name, age, gpa;
+d = foreach b generate nameagegpamap#'name' as name, nameagegpamap#'age' as age, nameagegpamap#'gpa' as gpa, nameagegpatuple.tage as tage, FLATTEN(nameagegpabag) as (bname, bage, bgpa);
+e = join c by name, d by name;
+f = group e by (d::bgpa);
+g = foreach f generate group, COUNT(e) as students;
+h = order g by group, students;
+store h into ':OUTPATH:';\,
+                        },
+# Tests 4 : Test various properties passed to ORCStorage
+                        {
+                        'num' => 4,
+                        'notmq' => 1,
+                        'pig' => q\
+register :HIVELIBDIR:/hive-common-:HIVEVERSION:.jar;
+register :HIVELIBDIR:/hive-serde-:HIVEVERSION:.jar;
+register :HIVELIBDIR:/hive-exec-:HIVEVERSION:-core.jar;
+register :HIVELIBDIR:/hive-shims-common-:HIVEVERSION:.jar;
+register :HIVELIBDIR:/hive-shims-common-secure-:HIVEVERSION:.jar;
+register :HIVELIBDIR:/hive-shims-:HIVESHIMSVERSION:-:HIVEVERSION:.jar;
+a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:float);
+store a into ':OUTPATH:.orc_params.intermediate' using OrcStorage('-c ZLIB -s 67108864 -r 100000 -b 1048576 -p true -v 0.12');
+b = load ':OUTPATH:.orc_params.intermediate' using OrcStorage();
+store b into ':OUTPATH:';\,
+                        'verify_pig_script' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:float);
+store a into ':OUTPATH:';\,
+                        }
+                        ]
+                }
+        ]
+};
+

Modified: pig/trunk/test/org/apache/pig/builtin/TestOrcStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/builtin/TestOrcStorage.java?rev=1614252&r1=1614251&r2=1614252&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/builtin/TestOrcStorage.java (original)
+++ pig/trunk/test/org/apache/pig/builtin/TestOrcStorage.java Tue Jul 29 05:06:09 2014
@@ -18,14 +18,16 @@
 package org.apache.pig.builtin;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 
 import java.io.File;
 import java.io.IOException;
 import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.Iterator;
-import java.util.Map;
+import java.util.List;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -33,22 +35,35 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.ql.io.orc.CompressionKind;
 import org.apache.hadoop.hive.ql.io.orc.OrcFile;
+import org.apache.hadoop.hive.ql.io.orc.OrcStruct;
 import org.apache.hadoop.hive.ql.io.orc.Reader;
 import org.apache.hadoop.hive.ql.io.orc.RecordReader;
+import org.apache.hadoop.hive.serde2.io.ByteWritable;
+import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
+import org.apache.hadoop.hive.serde2.io.ShortWritable;
+import org.apache.hadoop.hive.serde2.io.TimestampWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.io.BooleanWritable;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.FloatWritable;
 import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.pig.ExecType;
 import org.apache.pig.PigServer;
 import org.apache.pig.backend.executionengine.ExecException;
 import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
-import org.apache.pig.data.DataBag;
+import org.apache.pig.data.BinSedesTuple;
 import org.apache.pig.data.DataByteArray;
+import org.apache.pig.data.DefaultDataBag;
 import org.apache.pig.data.Tuple;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
 import org.apache.pig.test.Util;
 import org.joda.time.DateTime;
 import org.junit.After;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -57,16 +72,16 @@ public class TestOrcStorage {
 
     final private static String basedir = "test/org/apache/pig/builtin/orc/";
     final private static String outbasedir = System.getProperty("user.dir") + "/build/test/TestOrcStorage/";
-    
+
     private static String INPUT1 = outbasedir + "TestOrcStorage_1";
     private static String OUTPUT1 = outbasedir + "TestOrcStorage_2";
     private static String OUTPUT2 = outbasedir + "TestOrcStorage_3";
     private static String OUTPUT3 = outbasedir + "TestOrcStorage_4";
     private static String OUTPUT4 = outbasedir + "TestOrcStorage_5";
-    
+
     private static PigServer pigServer = null;
     private static FileSystem fs;
-    
+
     @Before
     public void setup() throws ExecException, IOException {
         pigServer = new PigServer(ExecType.LOCAL);
@@ -82,7 +97,7 @@ public class TestOrcStorage {
             OUTPUT4 = OUTPUT4.replace("\\", "/");
         }
     }
-    
+
     @After
     public void teardown() throws IOException {
         if(pigServer != null) {
@@ -90,12 +105,12 @@ public class TestOrcStorage {
         }
         deleteTestFiles();
     }
-    
+
     private void generateInputFiles() throws IOException {
         String[] input = {"65536\tworld", "1\thello"};
         Util.createLocalInputFile(INPUT1, input);
     }
-    
+
     private static void deleteTestFiles() throws IOException {
         Util.deleteDirectory(new File(outbasedir));
     }
@@ -109,30 +124,10 @@ public class TestOrcStorage {
                 "middle: (list: {(int1: int,string1: chararray)}),list: {(int1: int,string1: chararray)}," +
                 "map: map[(int1: int,string1: chararray)],ts: datetime,decimal1: bigdecimal}");
         Iterator<Tuple> iter = pigServer.openIterator("A");
-        int count=0;
-        Tuple t;
-        while (iter.hasNext()) {
-            t = (Tuple)iter.next();
-            assertEquals(t.size(), 14);
-            assertTrue(t.get(0) instanceof Boolean);
-            assertTrue(t.get(1) instanceof Integer);
-            assertTrue(t.get(2) instanceof Integer);
-            assertTrue(t.get(3) instanceof Integer);
-            assertTrue(t.get(4) instanceof Long);
-            assertTrue(t.get(5) instanceof Float);
-            assertTrue(t.get(6) instanceof Double);
-            assertTrue(t.get(7) instanceof DataByteArray);
-            assertTrue(t.get(8) instanceof String);
-            assertTrue(t.get(9) instanceof Tuple);
-            assertTrue(t.get(10) instanceof DataBag);
-            assertTrue(t.get(11) instanceof Map);
-            assertTrue(t.get(12) instanceof DateTime);
-            assertTrue(t.get(13) instanceof BigDecimal);
-            count++;
-        }
-        assertEquals(count, 7500);
+
+        verifyData(new Path(basedir + "orc-file-11-format.orc"), iter, fs, 7500);
     }
-    
+
     @Test
     public void testJoinWithPruning() throws Exception {
         pigServer.registerQuery("A = load '" + basedir + "orc-file-11-format.orc'" + " using OrcStorage();" );
@@ -159,7 +154,7 @@ public class TestOrcStorage {
         Path outputFilePath = new Path(new Path(OUTPUT1), "part-m-00000");
         Reader reader = OrcFile.createReader(fs, outputFilePath);
         assertEquals(reader.getNumberOfRows(), 2);
-        
+
         RecordReader rows = reader.rows(null);
         Object row = rows.next(null);
         StructObjectInspector soi = (StructObjectInspector)reader.getObjectInspector();
@@ -169,7 +164,7 @@ public class TestOrcStorage {
                 soi.getAllStructFieldRefs().get(1));
         assertEquals(intWritable.get(), 65536);
         assertEquals(text.toString(), "world");
-        
+
         row = rows.next(null);
         intWritable = (IntWritable)soi.getStructFieldData(row,
                 soi.getAllStructFieldRefs().get(0));
@@ -177,10 +172,10 @@ public class TestOrcStorage {
                 soi.getAllStructFieldRefs().get(1));
         assertEquals(intWritable.get(), 1);
         assertEquals(text.toString(), "hello");
-        
+
         // A bug in ORC InputFormat does not allow empty file in input directory
-        fs.delete(new Path(OUTPUT1, "_SUCCESS"));
-        
+        fs.delete(new Path(OUTPUT1, "_SUCCESS"), true);
+
         // Read the output file back
         pigServer.registerQuery("A = load '" + OUTPUT1 + "' using OrcStorage();");
         Schema s = pigServer.dumpSchema("A");
@@ -190,12 +185,12 @@ public class TestOrcStorage {
         assertEquals(t.size(), 2);
         assertEquals(t.get(0), 65536);
         assertEquals(t.get(1), "world");
-        
+
         t = iter.next();
         assertEquals(t.size(), 2);
         assertEquals(t.get(0), 1);
         assertEquals(t.get(1), "hello");
-        
+
         assertFalse(iter.hasNext());
     }
 
@@ -207,35 +202,168 @@ public class TestOrcStorage {
         pigServer.registerQuery("store B into '" + OUTPUT2 + "' using OrcStorage();");
         pigServer.registerQuery("store B into '" + OUTPUT3 +"' using OrcStorage('-c SNAPPY');");
         pigServer.executeBatch();
-        
+
         Path outputFilePath = new Path(new Path(OUTPUT2), "part-r-00000");
         Reader reader = OrcFile.createReader(fs, outputFilePath);
         assertEquals(reader.getNumberOfRows(), 2);
         assertEquals(reader.getCompression(), CompressionKind.ZLIB);
-        
-        outputFilePath = new Path(new Path(OUTPUT3), "part-r-00000");
-        reader = OrcFile.createReader(fs, outputFilePath);
+
+        Path outputFilePath2 = new Path(new Path(OUTPUT3), "part-r-00000");
+        reader = OrcFile.createReader(fs, outputFilePath2);
         assertEquals(reader.getNumberOfRows(), 2);
         assertEquals(reader.getCompression(), CompressionKind.SNAPPY);
+
+        verifyData(outputFilePath, outputFilePath2, fs, 2);
     }
-    
-    
+
+    @Test
+    public void testMultipleLoadStore() throws Exception {
+        pigServer.registerQuery("A = load '" + basedir + "orc-file-11-format.orc'" + " using OrcStorage();" );
+        pigServer.registerQuery("store A into '" + OUTPUT1 + "' using OrcStorage();");
+        pigServer.registerQuery("B = load '" + OUTPUT1 + "' using OrcStorage();");
+        verifyData(new Path(basedir + "orc-file-11-format.orc"), pigServer.openIterator("B"), fs, 7500);
+    }
+
     @Test
     public void testLoadStoreMoreDataType() throws Exception {
         pigServer.registerQuery("A = load '" + basedir + "orc-file-11-format.orc'" + " using OrcStorage();" );
         pigServer.registerQuery("B = foreach A generate boolean1..double1, '' as bytes1, string1..;");
         pigServer.store("B", OUTPUT4, "OrcStorage");
-        
+
         // A bug in ORC InputFormat does not allow empty file in input directory
-        fs.delete(new Path(OUTPUT4, "_SUCCESS"));
-        
+        fs.delete(new Path(OUTPUT4, "_SUCCESS"), true);
+
         pigServer.registerQuery("A = load '" + OUTPUT4 + "' using OrcStorage();" );
         Schema s = pigServer.dumpSchema("A");
         Iterator<Tuple> iter = pigServer.openIterator("A");
         Tuple t = iter.next();
         assertTrue(t.toString().startsWith("(false,1,1024,65536,9223372036854775807,1.0,-15.0," +
                 ",hi,({(1,bye),(2,sigh)}),{(3,good),(4,bad)},[],"));
-        assertTrue(t.get(12).toString().matches("2000-03-12T00:00:00.000.*"));
+        assertTrue(t.get(12).toString().matches("2000-03-12T15:00:00.000.*"));
         assertTrue(t.toString().endsWith(",12345678.6547456)"));
     }
+
+    private void verifyData(Path orcFile, Iterator<Tuple> iter, FileSystem fs, int expectedTotalRows) throws Exception {
+
+        int expectedRows = 0;
+        int actualRows = 0;
+        Reader orcReader = OrcFile.createReader(fs, orcFile);
+        ObjectInspector oi = orcReader.getObjectInspector();
+        StructObjectInspector soi = (StructObjectInspector) oi;
+
+        RecordReader reader = orcReader.rows(null);
+        Object row = null;
+
+        while (reader.hasNext()) {
+            row = reader.next(row);
+            expectedRows++;
+            List<?> orcRow = soi.getStructFieldsDataAsList(row);
+            if (!iter.hasNext()) {
+                break;
+            }
+            Tuple t = iter.next();
+            assertEquals(orcRow.size(), t.size());
+            actualRows++;
+
+            for (int i = 0; i < orcRow.size(); i++) {
+                Object expected = orcRow.get(i);
+                Object actual = t.get(i);
+                compareData(expected, actual);
+            }
+        }
+        assertFalse(iter.hasNext());
+        assertEquals(expectedRows, actualRows);
+        assertEquals(expectedTotalRows, actualRows);
+
+    }
+
+    private void verifyData(Path orcFile, Path pigOrcFile, FileSystem fs, int expectedTotalRows) throws Exception {
+
+        int expectedRows = 0;
+        int actualRows = 0;
+        Reader orcReaderExpected = OrcFile.createReader(fs, orcFile);
+        StructObjectInspector soiExpected = (StructObjectInspector) orcReaderExpected.getObjectInspector();
+        Reader orcReaderActual = OrcFile.createReader(fs, orcFile);
+        StructObjectInspector soiActual = (StructObjectInspector) orcReaderActual.getObjectInspector();
+
+        RecordReader readerExpected = orcReaderExpected.rows(null);
+        Object expectedRow = null;
+        RecordReader readerActual = orcReaderActual.rows(null);
+        Object actualRow = null;
+
+        while (readerExpected.hasNext()) {
+            expectedRow = readerExpected.next(expectedRow);
+            expectedRows++;
+            List<?> orcRowExpected = soiExpected.getStructFieldsDataAsList(expectedRow);
+            if (!readerActual.hasNext()) {
+                break;
+            }
+            actualRow = readerActual.next(actualRow);
+            actualRows++;
+            List<?> orcRowActual = soiActual.getStructFieldsDataAsList(actualRow);
+            assertEquals(orcRowExpected.size(), orcRowActual.size());
+
+            for (int i = 0; i < orcRowExpected.size(); i++) {
+                assertEquals(orcRowExpected.get(i), orcRowActual.get(i));
+            }
+        }
+        assertFalse(readerActual.hasNext());
+        assertEquals(expectedRows, actualRows);
+        assertEquals(expectedTotalRows, actualRows);
+
+    }
+
+    private void compareData(Object expected, Object actual) {
+        if (expected instanceof Text) {
+            assertEquals(String.class, actual.getClass());
+            assertEquals(expected.toString(), actual);
+        } else if (expected instanceof ShortWritable) {
+            assertEquals(Integer.class, actual.getClass());
+            assertEquals((int)((ShortWritable) expected).get(), actual);
+        } else if (expected instanceof IntWritable) {
+            assertEquals(Integer.class, actual.getClass());
+            assertEquals(((IntWritable) expected).get(), actual);
+        } else if (expected instanceof LongWritable) {
+            assertEquals(Long.class, actual.getClass());
+            assertEquals(((LongWritable) expected).get(), actual);
+        } else if (expected instanceof FloatWritable) {
+            assertEquals(Float.class, actual.getClass());
+            assertEquals(((FloatWritable) expected).get(), actual);
+        } else if (expected instanceof HiveDecimalWritable) {
+            assertEquals(BigDecimal.class, actual.getClass());
+            assertEquals(((HiveDecimalWritable) expected).toString(), actual.toString());
+        } else if (expected instanceof DoubleWritable) {
+            assertEquals(Double.class, actual.getClass());
+            assertEquals(((DoubleWritable) expected).get(), actual);
+        } else if (expected instanceof BooleanWritable) {
+            assertEquals(Boolean.class, actual.getClass());
+            assertEquals(((BooleanWritable) expected).get(), actual);
+        } else if (expected instanceof TimestampWritable) {
+            assertEquals(DateTime.class, actual.getClass());
+            assertEquals(((TimestampWritable) expected).getTimestamp().getTime(),
+                    ((DateTime) actual).getMillis());
+        } else if (expected instanceof BytesWritable) {
+            assertEquals(DataByteArray.class, actual.getClass());
+            BytesWritable bw = (BytesWritable) expected;
+            assertEquals(new DataByteArray(bw.getBytes(), 0, bw.getLength()), actual);
+        } else if (expected instanceof ByteWritable) {
+            assertEquals(Integer.class, actual.getClass());
+            assertEquals((int) ((ByteWritable) expected).get(), actual);
+        } else if (expected instanceof OrcStruct) {
+            assertEquals(BinSedesTuple.class, actual.getClass());
+            // TODO: compare actual values. No getters in OrcStruct
+        } else if (expected instanceof ArrayList) {
+            assertEquals(DefaultDataBag.class, actual.getClass());
+            // TODO: compare actual values. No getters in OrcStruct
+        } else if (expected instanceof HashMap) {
+            assertEquals(HashMap.class, actual.getClass());
+            assertEquals(((HashMap) expected).size(), ((HashMap) actual).size());
+            // TODO: compare actual values. No getters in OrcStruct
+        } else if (expected == null) {
+            assertEquals(expected, actual);
+        } else {
+            Assert.fail("Unknown object type: " + expected.getClass().getName());
+        }
+    }
+
 }