You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2014/07/29 07:06:10 UTC
svn commit: r1614252 - in /pig/trunk: ./ src/org/apache/pig/builtin/
src/org/apache/pig/impl/util/orc/ test/e2e/pig/ test/e2e/pig/conf/
test/e2e/pig/drivers/ test/e2e/pig/tests/ test/org/apache/pig/builtin/
Author: rohini
Date: Tue Jul 29 05:06:09 2014
New Revision: 1614252
URL: http://svn.apache.org/r1614252
Log:
PIG-4077: Some fixes and e2e test for OrcStorage (rohini)
Added:
pig/trunk/test/e2e/pig/tests/orc.conf
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/builtin/OrcStorage.java
pig/trunk/src/org/apache/pig/impl/util/orc/OrcUtils.java
pig/trunk/test/e2e/pig/build.xml
pig/trunk/test/e2e/pig/conf/default.conf
pig/trunk/test/e2e/pig/conf/local.conf
pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm
pig/trunk/test/org/apache/pig/builtin/TestOrcStorage.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1614252&r1=1614251&r2=1614252&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Jul 29 05:06:09 2014
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
IMPROVEMENTS
+PIG-4077: Some fixes and e2e test for OrcStorage (rohini)
+
PIG-4054: Do not create job.jar when submitting job (daijy)
PIG-4047: Break up pig withouthadoop and fat jar (daijy)
Modified: pig/trunk/src/org/apache/pig/builtin/OrcStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/builtin/OrcStorage.java?rev=1614252&r1=1614251&r2=1614252&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/builtin/OrcStorage.java (original)
+++ pig/trunk/src/org/apache/pig/builtin/OrcStorage.java Tue Jul 29 05:06:09 2014
@@ -83,9 +83,9 @@ import org.apache.pig.impl.util.orc.OrcU
* <ul>
* <li><code>-s, --stripeSize</code> Set the stripe size for the file
* <li><code>-r, --rowIndexStride</code> Set the distance between entries in the row index
- * <li><code>-b, --bufferSize</code> The size of the memory buffers used for compressing and storing the
+ * <li><code>-b, --bufferSize</code> The size of the memory buffers used for compressing and storing the
* stripe in memory
- * <li><code>-p, --blockPadding</code> Sets whether the HDFS blocks are padded to prevent stripes
+ * <li><code>-p, --blockPadding</code> Sets whether the HDFS blocks are padded to prevent stripes
* from straddling blocks
* <li><code>-c, --compress</code> Sets the generic compression that is used to compress the data.
* Valid codecs are: NONE, ZLIB, SNAPPY, LZO
@@ -105,16 +105,16 @@ public class OrcStorage extends LoadFunc
private Integer bufferSize;
private Boolean blockPadding;
private CompressionKind compress;
- private org.apache.hadoop.hive.ql.io.orc.OrcFile.Version version;
-
+ private Version version;
+
private static final Options validOptions;
private final CommandLineParser parser = new GnuParser();
- protected final Log log = LogFactory.getLog(getClass());
+ protected final static Log log = LogFactory.getLog(OrcStorage.class);
protected boolean[] mRequiredColumns = null;
-
+
private static final String SchemaSignatureSuffix = "_schema";
private static final String RequiredColumnsSuffix = "_columns";
-
+
static {
validOptions = new Options();
validOptions.addOption("s", "stripeSize", true,
@@ -131,10 +131,10 @@ public class OrcStorage extends LoadFunc
validOptions.addOption("v", "version", true,
"Sets the version of the file that will be written");
}
-
+
public OrcStorage() {
}
-
+
public OrcStorage(String options) {
String[] optsArr = options.split(" ");
try {
@@ -200,7 +200,7 @@ public class OrcStorage extends LoadFunc
}
if (version!=null) {
job.getConfiguration().set(HiveConf.ConfVars.HIVE_ORC_WRITE_FORMAT.varname,
- version.toString());
+ version.getName());
}
}
FileOutputFormat.setOutputPath(job, new Path(location));
@@ -241,7 +241,7 @@ public class OrcStorage extends LoadFunc
public void setStoreFuncUDFContextSignature(String signature) {
this.signature = signature;
}
-
+
@Override
public void setUDFContextSignature(String signature) {
this.signature = signature;
@@ -261,20 +261,22 @@ public class OrcStorage extends LoadFunc
Properties p = UDFContext.getUDFContext().getUDFProperties(this.getClass());
if (!UDFContext.getUDFContext().isFrontend()) {
typeInfo = (TypeInfo)ObjectSerializer.deserialize(p.getProperty(signature + SchemaSignatureSuffix));
+ } else if (typeInfo == null) {
+ typeInfo = getTypeInfo(location, job);
}
- if (typeInfo!=null && oi==null) {
+ if (typeInfo != null && oi == null) {
oi = OrcStruct.createObjectInspector(typeInfo);
}
if (!UDFContext.getUDFContext().isFrontend() &&
p.getProperty(signature+RequiredColumnsSuffix)!=null) {
mRequiredColumns = (boolean[])ObjectSerializer.deserialize(p.getProperty(signature+RequiredColumnsSuffix));
job.getConfiguration().setBoolean(ColumnProjectionUtils.READ_ALL_COLUMNS, false);
- job.getConfiguration().set(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR,
+ job.getConfiguration().set(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR,
getReqiredColumnIdString(mRequiredColumns));
}
FileInputFormat.setInputPaths(job, location);
}
-
+
private static String getReqiredColumnIdString(boolean[] requiredColumns) {
String result = "";
for (int i=0;i<requiredColumns.length;i++) {
@@ -308,7 +310,7 @@ public class OrcStorage extends LoadFunc
return null;
}
Object value = in.getCurrentValue();
-
+
Tuple t = (Tuple)OrcUtils.convertOrcToPig(value, oi, mRequiredColumns);
return t;
} catch (InterruptedException e) {
@@ -318,7 +320,7 @@ public class OrcStorage extends LoadFunc
PigException.REMOTE_ENVIRONMENT, e);
}
}
-
+
private static Path getFirstFile(String location, FileSystem fs) throws IOException {
String[] locations = getPathStrings(location);
Path[] paths = new Path[locations.length];
@@ -327,8 +329,11 @@ public class OrcStorage extends LoadFunc
}
List<FileStatus> statusList = new ArrayList<FileStatus>();
for (int i = 0; i < paths.length; ++i) {
- for (FileStatus tempf : fs.globStatus(paths[i])) {
- statusList.add(tempf);
+ FileStatus[] files = fs.globStatus(paths[i]);
+ if (files != null) {
+ for (FileStatus tempf : files) {
+ statusList.add(tempf);
+ }
}
}
FileStatus[] statusArray = (FileStatus[]) statusList
@@ -340,26 +345,46 @@ public class OrcStorage extends LoadFunc
private static TypeInfo getTypeInfoFromLocation(String location, Job job) throws IOException {
FileSystem fs = FileSystem.get(job.getConfiguration());
Path path = getFirstFile(location, fs);
- if (path==null) {
- throw new IOException("Cannot find any ORC files from " + location);
+ if (path == null) {
+ log.info("Cannot find any ORC files from " + location +
+ ". Probably multiple load store in script.");
+ return null;
}
Reader reader = OrcFile.createReader(fs, path);
ObjectInspector oip = (ObjectInspector)reader.getObjectInspector();
return TypeInfoUtils.getTypeInfoFromObjectInspector(oip);
}
-
+
@Override
public ResourceSchema getSchema(String location, Job job)
throws IOException {
- if (typeInfo==null) {
- typeInfo = getTypeInfoFromLocation(location, job);
- Properties p = UDFContext.getUDFContext().getUDFProperties(this.getClass());
- p.setProperty(signature + SchemaSignatureSuffix, ObjectSerializer.serialize(typeInfo));
+ if (typeInfo == null) {
+ typeInfo = getTypeInfo(location, job);
+ // still null means case of multiple load store
+ if (typeInfo == null) {
+ return null;
+ }
}
+
ResourceFieldSchema fs = OrcUtils.getResourceFieldSchema(typeInfo);
return fs.getSchema();
}
+ private TypeInfo getTypeInfo(String location, Job job) throws IOException {
+ Properties p = UDFContext.getUDFContext().getUDFProperties(this.getClass());
+ String serializedStr = p.getProperty(signature + SchemaSignatureSuffix);
+ TypeInfo typeInfo = null;
+ if (serializedStr != null) {
+ typeInfo = (TypeInfo) ObjectSerializer.deserialize(serializedStr);
+ } else {
+ typeInfo = getTypeInfoFromLocation(location, job);
+ }
+ if (typeInfo != null) {
+ p.setProperty(signature + SchemaSignatureSuffix, ObjectSerializer.serialize(typeInfo));
+ }
+ return typeInfo;
+ }
+
@Override
public ResourceStatistics getStatistics(String location, Job job)
throws IOException {
Modified: pig/trunk/src/org/apache/pig/impl/util/orc/OrcUtils.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/util/orc/OrcUtils.java?rev=1614252&r1=1614251&r2=1614252&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/util/orc/OrcUtils.java (original)
+++ pig/trunk/src/org/apache/pig/impl/util/orc/OrcUtils.java Tue Jul 29 05:06:09 2014
@@ -20,6 +20,7 @@ package org.apache.pig.impl.util.orc;
import java.io.IOException;
import java.math.BigDecimal;
import java.math.BigInteger;
+import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
@@ -27,8 +28,8 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.hive.common.type.HiveDecimal;
-import org.apache.hadoop.hive.serde2.io.DateWritable;
import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
+import org.apache.hadoop.hive.serde2.io.TimestampWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -36,16 +37,10 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableBooleanObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableDoubleObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableFloatObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableIntObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableLongObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableStringObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableTimestampObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
@@ -66,7 +61,10 @@ import org.joda.time.DateTime;
public class OrcUtils {
public static Object convertOrcToPig(Object obj, ObjectInspector oi, boolean[] includedColumns) {
- Object result;
+ Object result = null;
+ if (obj == null) {
+ return result;
+ }
switch (oi.getCategory()) {
case PRIMITIVE:
PrimitiveObjectInspector poi = (PrimitiveObjectInspector)oi;
@@ -112,9 +110,12 @@ public class OrcUtils {
}
return result;
}
-
+
public static Object getPrimaryFromOrc(Object obj, PrimitiveObjectInspector poi) {
- Object result;
+ Object result = null;
+ if (obj == null) {
+ return result;
+ }
switch (poi.getPrimitiveCategory()) {
case FLOAT:
case DOUBLE:
@@ -131,9 +132,9 @@ public class OrcUtils {
result = (int)(Short)poi.getPrimitiveJavaObject(obj);
break;
case BINARY:
- byte[] r = new byte[((BytesWritable)obj).getLength()];
- System.arraycopy(((BytesWritable)obj).getBytes(), 0, r, 0, ((BytesWritable)obj).getLength());
- result = new DataByteArray(r);
+ BytesWritable bw = (BytesWritable) obj;
+ // Make a copy
+ result = new DataByteArray(bw.getBytes(), 0, bw.getLength());
break;
case TIMESTAMP:
java.sql.Timestamp origTimeStamp = (java.sql.Timestamp)poi.getPrimitiveJavaObject(obj);
@@ -144,7 +145,7 @@ public class OrcUtils {
result = new DateTime(origDate.getTime());
break;
case DECIMAL:
- org.apache.hadoop.hive.common.type.HiveDecimal origDecimal =
+ org.apache.hadoop.hive.common.type.HiveDecimal origDecimal =
(org.apache.hadoop.hive.common.type.HiveDecimal)poi.getPrimitiveJavaObject(obj);
result = origDecimal.bigDecimalValue();
break;
@@ -154,50 +155,7 @@ public class OrcUtils {
}
return result;
}
-
- public static Object getPrimaryFromPig(Object obj, PrimitiveObjectInspector poi) throws IOException {
- Object result=null;
- switch (poi.getPrimitiveCategory()) {
- case FLOAT:
- result = ((SettableFloatObjectInspector)poi).create((Float)obj);
- break;
- case DOUBLE:
- result = ((SettableDoubleObjectInspector)poi).create((Double)obj);
- break;
- case BOOLEAN:
- result = ((SettableBooleanObjectInspector)poi).create((Boolean)obj);
- break;
- case INT:
- result = ((SettableIntObjectInspector)poi).create((Integer)obj);
- break;
- case LONG:
- result = ((SettableLongObjectInspector)poi).create((Long)obj);
- break;
- case STRING:
- result = ((SettableStringObjectInspector)poi).create((String)obj);
- break;
- case TIMESTAMP:
- DateTime origDateTime = (DateTime)obj;
- result = ((SettableTimestampObjectInspector)poi).create(new java.sql.Timestamp(origDateTime.getMillis()));
- break;
- case DECIMAL:
- if (obj instanceof BigDecimal) {
- result = org.apache.hadoop.hive.common.type.HiveDecimal.create((BigDecimal)obj);
- } else if (obj instanceof BigInteger) {
- result = org.apache.hadoop.hive.common.type.HiveDecimal.create((BigInteger)obj);
- } else {
- throw new IOException("Should never happen, except BigDecimal/BigInteger, but get " + obj.getClass());
- }
- break;
- case DATE:
- case BYTE:
- case SHORT:
- case BINARY:
- throw new IOException("Should never happen, expect a binary, which is not valid Pig data type");
- }
- return result;
- }
-
+
public static ResourceFieldSchema getResourceFieldSchema(TypeInfo ti) throws IOException {
ResourceFieldSchema fieldSchema = new ResourceFieldSchema();
ResourceFieldSchema[] innerFs;
@@ -279,10 +237,10 @@ public class OrcUtils {
}
break;
}
-
+
return fieldSchema;
}
-
+
public static TypeInfo getTypeInfo(ResourceFieldSchema fs) throws IOException {
TypeInfo ti;
switch (fs.getType()) {
@@ -308,10 +266,12 @@ public class OrcUtils {
break;
case DataType.MAP:
ti = new MapTypeInfo();
- if (fs.getSchema()==null || fs.getSchema().getFields().length!=1) {
- throw new IOException("Wrong map inner schema");
+ TypeInfo valueField;
+ if (fs.getSchema() == null || fs.getSchema().getFields().length != 1) {
+ valueField = TypeInfoFactory.binaryTypeInfo;
+ } else {
+ valueField = getTypeInfo(fs.getSchema().getFields()[0]);
}
- TypeInfo valueField = getTypeInfo(fs.getSchema().getFields()[0]);
((MapTypeInfo)ti).setMapKeyTypeInfo(TypeInfoFactory.stringTypeInfo);
((MapTypeInfo)ti).setMapValueTypeInfo(valueField);
break;
@@ -342,13 +302,16 @@ public class OrcUtils {
case DataType.BIGINTEGER:
ti = TypeInfoFactory.decimalTypeInfo;
break;
+ case DataType.BYTEARRAY:
+ ti = TypeInfoFactory.binaryTypeInfo;
+ break;
default:
throw new IllegalArgumentException("Unknown data type " +
DataType.findTypeName(fs.getType()));
}
return ti;
}
-
+
static class Field implements StructField {
private final String name;
private final ObjectInspector inspector;
@@ -380,7 +343,7 @@ public class OrcUtils {
return null;
}
}
-
+
static class PigStructInspector extends StructObjectInspector {
private List<StructField> fields;
@@ -471,7 +434,7 @@ public class OrcUtils {
}
}
}
-
+
static class PigMapObjectInspector implements MapObjectInspector {
private ObjectInspector key;
private ObjectInspector value;
@@ -480,7 +443,7 @@ public class OrcUtils {
key = createObjectInspector(info.getMapKeyTypeInfo());
value = createObjectInspector(info.getMapValueTypeInfo());
}
-
+
@Override
public ObjectInspector getMapKeyObjectInspector() {
return key;
@@ -528,7 +491,7 @@ public class OrcUtils {
}
}
}
-
+
static class PigListObjectInspector implements ListObjectInspector {
private ObjectInspector child;
private Object cachedObject;
@@ -598,25 +561,46 @@ public class OrcUtils {
}
}
}
-
- static class JodaDateObjectInspector extends
- AbstractPrimitiveJavaObjectInspector implements DateObjectInspector {
- protected JodaDateObjectInspector() {
- super(TypeInfoFactory.dateTypeInfo);
+ static class PigDataByteArrayObjectInspector extends AbstractPrimitiveJavaObjectInspector
+ implements BinaryObjectInspector {
+
+ PigDataByteArrayObjectInspector() {
+ super(TypeInfoFactory.binaryTypeInfo);
}
@Override
- public DateWritable getPrimitiveWritableObject(Object o) {
- return o == null ? null : new DateWritable(new java.sql.Date(((DateTime)o).getMillis()));
+ public BytesWritable getPrimitiveWritableObject(Object o) {
+ return o == null ? null : (o instanceof DataByteArray
+ ? new BytesWritable(((DataByteArray) o).get())
+ : new BytesWritable((byte[]) o));
}
@Override
- public java.sql.Date getPrimitiveJavaObject(Object o) {
- return o == null ? null : new java.sql.Date(((DateTime)o).getMillis());
+ public byte[] getPrimitiveJavaObject(Object o) {
+ return ((DataByteArray) o).get();
}
+
}
-
+
+ static class PigJodaTimeStampObjectInspector extends
+ AbstractPrimitiveJavaObjectInspector implements TimestampObjectInspector {
+
+ protected PigJodaTimeStampObjectInspector() {
+ super(TypeInfoFactory.timestampTypeInfo);
+ }
+
+ @Override
+ public TimestampWritable getPrimitiveWritableObject(Object o) {
+ return o == null ? null : new TimestampWritable(new Timestamp(((DateTime)o).getMillis()));
+ }
+
+ @Override
+ public Timestamp getPrimitiveJavaObject(Object o) {
+ return o == null ? null : new Timestamp(((DateTime)o).getMillis());
+ }
+ }
+
static class PigDecimalObjectInspector extends
AbstractPrimitiveJavaObjectInspector implements HiveDecimalObjectInspector {
@@ -642,7 +626,7 @@ public class OrcUtils {
}
}
}
-
+
public static ObjectInspector createObjectInspector(TypeInfo info) {
switch (info.getCategory()) {
case PRIMITIVE:
@@ -660,14 +644,15 @@ public class OrcUtils {
case STRING:
return PrimitiveObjectInspectorFactory.javaStringObjectInspector;
case TIMESTAMP:
- return new JodaDateObjectInspector();
+ return new PigJodaTimeStampObjectInspector();
case DECIMAL:
return new PigDecimalObjectInspector();
+ case BINARY:
+ return new PigDataByteArrayObjectInspector();
case DATE:
case VARCHAR:
case BYTE:
case SHORT:
- case BINARY:
throw new IllegalArgumentException("Should never happen, " +
(((PrimitiveTypeInfo) info).getPrimitiveCategory()) +
"is not valid Pig primitive data type");
Modified: pig/trunk/test/e2e/pig/build.xml
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/build.xml?rev=1614252&r1=1614251&r2=1614252&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/build.xml (original)
+++ pig/trunk/test/e2e/pig/build.xml Tue Jul 29 05:06:09 2014
@@ -25,6 +25,12 @@
value="${pig.dir}/build/ivy/lib/Pig/jython-standalone-${jython.version}.jar"/>
<property name="jruby.jar"
value="${pig.dir}/build/ivy/lib/Pig/jruby-complete-${jruby.version}.jar"/>
+ <property name="hive.lib.dir"
+ value="${pig.dir}/build/ivy/lib/Pig"/>
+
+ <condition property="hive.hadoop.shims.version" value="0.23" else="0.20S">
+ <equals arg1="${hadoopversion}" arg2="23" />
+ </condition>
<!-- Separate property name for udfs' build.xml -->
<property name="udf.dir" value="${basedir}/udfs"/>
@@ -275,6 +281,9 @@
<env key="PH_CLUSTER_BIN" value="${harness.cluster.bin}"/>
<env key="PH_JYTHON_JAR" value="${jython.jar}"/>
<env key="PH_JRUBY_JAR" value="${jruby.jar}"/>
+ <env key="PH_HIVE_LIB_DIR" value="${hive.lib.dir}"/>
+ <env key="PH_HIVE_VERSION" value="${hive.version}"/>
+ <env key="PH_HIVE_SHIMS_VERSION" value="${hive.hadoop.shims.version}"/>
<env key="JRUBY_OPTS" value="--1.9"/>
<env key="HARNESS_CONF" value="${harness.conf.file}"/>
<env key="HADOOP_HOME" value="${harness.hadoop.home}"/>
@@ -306,6 +315,7 @@
<arg value="${test.location}/tests/bigdata.conf"/>
<arg value="${test.location}/tests/grunt.conf"/>
<arg value="${test.location}/tests/macro.conf"/>
+ <arg value="${test.location}/tests/orc.conf"/>
<arg value="${test.location}/tests/hcat.conf"/>
</exec>
</target>
Modified: pig/trunk/test/e2e/pig/conf/default.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/conf/default.conf?rev=1614252&r1=1614251&r2=1614252&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/conf/default.conf (original)
+++ pig/trunk/test/e2e/pig/conf/default.conf Tue Jul 29 05:06:09 2014
@@ -61,6 +61,10 @@ $cfg = {
#HADOOP
, 'mapredjars' => "$ENV{PH_ROOT}/lib"
+ #HIVE
+ , 'hivelibdir' => "$ENV{PH_HIVE_LIB_DIR}"
+ , 'hiveversion' => "$ENV{PH_HIVE_VERSION}"
+ , 'hiveshimsversion' => "$ENV{PH_HIVE_SHIMS_VERSION}"
, 'userhomePath' => "$ENV{HOME}"
,'local.bin' => '/usr/bin'
Modified: pig/trunk/test/e2e/pig/conf/local.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/conf/local.conf?rev=1614252&r1=1614251&r2=1614252&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/conf/local.conf (original)
+++ pig/trunk/test/e2e/pig/conf/local.conf Tue Jul 29 05:06:09 2014
@@ -57,6 +57,11 @@ $cfg = {
, 'mapredjars' => "$ENV{PH_ROOT}/lib"
, 'hadoopHome' => "$ENV{PH_ROOT}/lib"
+ #HIVE
+ , 'hivelibdir' => "$ENV{PH_HIVE_LIB_DIR}"
+ , 'hiveversion' => "$ENV{PH_HIVE_VERSION}"
+ , 'hiveshimsversion' => "$ENV{PH_HIVE_SHIMS_VERSION}"
+
, 'userhomePath' => "$ENV{HOME}"
,'local.bin' => '/usr/bin'
# NB: we provide the default there as a fallback in case if there is no corresponding ENV key:
Modified: pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm?rev=1614252&r1=1614251&r2=1614252&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm (original)
+++ pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm Tue Jul 29 05:06:09 2014
@@ -78,6 +78,9 @@ sub replaceParameters
$cmd =~ s/:RUNID:/$testCmd->{'UID'}/g;
$cmd =~ s/:USRHOMEPATH:/$testCmd->{'userhomePath'}/g;
$cmd =~ s/:MAPREDJARS:/$testCmd->{'mapredjars'}/g;
+ $cmd =~ s/:HIVELIBDIR:/$testCmd->{'hivelibdir'}/g;
+ $cmd =~ s/:HIVEVERSION:/$testCmd->{'hiveversion'}/g;
+ $cmd =~ s/:HIVESHIMSVERSION:/$testCmd->{'hiveshimsversion'}/g;
$cmd =~ s/:SCRIPTHOMEPATH:/$testCmd->{'scriptPath'}/g;
$cmd =~ s/:DBUSER:/$testCmd->{'dbuser'}/g;
$cmd =~ s/:DBNAME:/$testCmd->{'dbdb'}/g;
Added: pig/trunk/test/e2e/pig/tests/orc.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/tests/orc.conf?rev=1614252&view=auto
==============================================================================
--- pig/trunk/test/e2e/pig/tests/orc.conf (added)
+++ pig/trunk/test/e2e/pig/tests/orc.conf Tue Jul 29 05:06:09 2014
@@ -0,0 +1,109 @@
+$cfg = {
+ 'driver' => 'Pig',
+ 'nummachines' => 5,
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
+
+ 'groups' => [
+ {
+ 'name' => 'Orc',
+ 'tests' => [
+# Test 1: Load (primitive) from PigStorage and store into OrcStorage
+# Also tests multiple load stores in same script
+ {
+ 'num' => 1,
+ 'notmq' => 1,
+ 'pig' => q\
+register :HIVELIBDIR:/hive-common-:HIVEVERSION:.jar;
+register :HIVELIBDIR:/hive-serde-:HIVEVERSION:.jar;
+register :HIVELIBDIR:/hive-exec-:HIVEVERSION:-core.jar;
+register :HIVELIBDIR:/hive-shims-common-:HIVEVERSION:.jar;
+register :HIVELIBDIR:/hive-shims-common-secure-:HIVEVERSION:.jar;
+register :HIVELIBDIR:/hive-shims-:HIVESHIMSVERSION:-:HIVEVERSION:.jar;
+a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:float);
+store a into ':OUTPATH:.intermediate' using OrcStorage();
+b = load ':OUTPATH:.intermediate' using OrcStorage();
+describe b;
+c = filter b by age < 30;
+store c into ':OUTPATH:';\,
+ 'verify_pig_script' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:float);
+b = filter a by age < 30;
+store b into ':OUTPATH:';\,
+ },
+# Test 2: Load (complex) from PigStorage and store into OrcStorage
+ {
+ 'num' => 2,
+ 'notmq' => 1,
+ 'pig' => q\
+register :HIVELIBDIR:/hive-common-:HIVEVERSION:.jar;
+register :HIVELIBDIR:/hive-serde-:HIVEVERSION:.jar;
+register :HIVELIBDIR:/hive-exec-:HIVEVERSION:-core.jar;
+register :HIVELIBDIR:/hive-shims-common-:HIVEVERSION:.jar;
+register :HIVELIBDIR:/hive-shims-common-secure-:HIVEVERSION:.jar;
+register :HIVELIBDIR:/hive-shims-:HIVESHIMSVERSION:-:HIVEVERSION:.jar;
+a = load ':INPATH:/singlefile/studentcomplextab10k' as (nameagegpamap:map[], nameagegpatuple:tuple(tname:chararray, tage:int, tgpa:float), nameagegpabag:bag{t:tuple(bname:chararray, bage:int, bgpa:float)});
+store a into ':OUTPATH:.intermediate' using OrcStorage();
+b = load ':OUTPATH:.intermediate' using OrcStorage();
+store b into ':OUTPATH:';\,
+ 'verify_pig_script' => q\a = load ':INPATH:/singlefile/studentcomplextab10k' as (nameagegpamap:map[], nameagegpatuple:tuple(tname:chararray, tage:int, tgpa:float), nameagegpabag:bag{t:tuple(bname:chararray, bage:int, bgpa:float)});
+store a into ':OUTPATH:';\,
+ },
+# Test 3: Aggregation test using two ORCStorage datasets
+# Also incorporates handling bytearrays
+ {
+ 'num' => 3,
+ 'notmq' => 1,
+ 'pig' => q\register :HIVELIBDIR:/hive-common-:HIVEVERSION:.jar;
+register :HIVELIBDIR:/hive-serde-:HIVEVERSION:.jar;
+register :HIVELIBDIR:/hive-exec-:HIVEVERSION:-core.jar;
+register :HIVELIBDIR:/hive-shims-common-:HIVEVERSION:.jar;
+register :HIVELIBDIR:/hive-shims-common-secure-:HIVEVERSION:.jar;
+register :HIVELIBDIR:/hive-shims-:HIVESHIMSVERSION:-:HIVEVERSION:.jar;
+a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:float);
+store a into ':OUTPATH:.simple.intermediate' using OrcStorage();
+b = load ':INPATH:/singlefile/studentcomplextab10k' as (nameagegpamap:map[], nameagegpatuple:tuple(tname:chararray, tage:int, tgpa:float), nameagegpabag:bag{t:tuple(bname:chararray, bage:int, bgpa:float)}, nameagegpamap_name:chararray, nameagegpamap_age:int, nameagegpamap_gpa:float);
+store b into ':OUTPATH:.complex.intermediate' using OrcStorage();
+c = load ':OUTPATH:.simple.intermediate' using OrcStorage();
+describe c;
+d = load ':OUTPATH:.complex.intermediate' using OrcStorage();
+describe d;
+e = foreach c generate name, age, gpa;
+f = foreach d generate nameagegpamap#'name' as name, nameagegpamap#'age' as age, nameagegpamap#'gpa' as gpa, nameagegpatuple.tage as tage, FLATTEN(nameagegpabag) as (bname, bage, bgpa);
+g = join e by name, f by name;
+h = group g by (f::bgpa);
+j = foreach h generate group, COUNT(g) as students;
+k = order j by group, students;
+store k into ':OUTPATH:';\,
+ 'verify_pig_script' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:float);
+b = load ':INPATH:/singlefile/studentcomplextab10k' as (nameagegpamap:map[], nameagegpatuple:tuple(tname:chararray, tage:int, tgpa:float), nameagegpabag:bag{t:tuple(bname:chararray, bage:int, bgpa:float)}, nameagegpamap_name:chararray, nameagegpamap_age:int, nameagegpamap_gpa:float);
+c = foreach a generate name, age, gpa;
+d = foreach b generate nameagegpamap#'name' as name, nameagegpamap#'age' as age, nameagegpamap#'gpa' as gpa, nameagegpatuple.tage as tage, FLATTEN(nameagegpabag) as (bname, bage, bgpa);
+e = join c by name, d by name;
+f = group e by (d::bgpa);
+g = foreach f generate group, COUNT(e) as students;
+h = order g by group, students;
+store h into ':OUTPATH:';\,
+ },
+# Tests 4 : Test various properties passed to ORCStorage
+ {
+ 'num' => 4,
+ 'notmq' => 1,
+ 'pig' => q\
+register :HIVELIBDIR:/hive-common-:HIVEVERSION:.jar;
+register :HIVELIBDIR:/hive-serde-:HIVEVERSION:.jar;
+register :HIVELIBDIR:/hive-exec-:HIVEVERSION:-core.jar;
+register :HIVELIBDIR:/hive-shims-common-:HIVEVERSION:.jar;
+register :HIVELIBDIR:/hive-shims-common-secure-:HIVEVERSION:.jar;
+register :HIVELIBDIR:/hive-shims-:HIVESHIMSVERSION:-:HIVEVERSION:.jar;
+a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:float);
+store a into ':OUTPATH:.orc_params.intermediate' using OrcStorage('-c ZLIB -s 67108864 -r 100000 -b 1048576 -p true -v 0.12');
+b = load ':OUTPATH:.orc_params.intermediate' using OrcStorage();
+store b into ':OUTPATH:';\,
+ 'verify_pig_script' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:float);
+store a into ':OUTPATH:';\,
+ }
+ ]
+ }
+ ]
+};
+
Modified: pig/trunk/test/org/apache/pig/builtin/TestOrcStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/builtin/TestOrcStorage.java?rev=1614252&r1=1614251&r2=1614252&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/builtin/TestOrcStorage.java (original)
+++ pig/trunk/test/org/apache/pig/builtin/TestOrcStorage.java Tue Jul 29 05:06:09 2014
@@ -18,14 +18,16 @@
package org.apache.pig.builtin;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.IOException;
import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.HashMap;
import java.util.Iterator;
-import java.util.Map;
+import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -33,22 +35,35 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.io.orc.CompressionKind;
import org.apache.hadoop.hive.ql.io.orc.OrcFile;
+import org.apache.hadoop.hive.ql.io.orc.OrcStruct;
import org.apache.hadoop.hive.ql.io.orc.Reader;
import org.apache.hadoop.hive.ql.io.orc.RecordReader;
+import org.apache.hadoop.hive.serde2.io.ByteWritable;
+import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
+import org.apache.hadoop.hive.serde2.io.ShortWritable;
+import org.apache.hadoop.hive.serde2.io.TimestampWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.io.BooleanWritable;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
-import org.apache.pig.data.DataBag;
+import org.apache.pig.data.BinSedesTuple;
import org.apache.pig.data.DataByteArray;
+import org.apache.pig.data.DefaultDataBag;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.test.Util;
import org.joda.time.DateTime;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -57,16 +72,16 @@ public class TestOrcStorage {
final private static String basedir = "test/org/apache/pig/builtin/orc/";
final private static String outbasedir = System.getProperty("user.dir") + "/build/test/TestOrcStorage/";
-
+
private static String INPUT1 = outbasedir + "TestOrcStorage_1";
private static String OUTPUT1 = outbasedir + "TestOrcStorage_2";
private static String OUTPUT2 = outbasedir + "TestOrcStorage_3";
private static String OUTPUT3 = outbasedir + "TestOrcStorage_4";
private static String OUTPUT4 = outbasedir + "TestOrcStorage_5";
-
+
private static PigServer pigServer = null;
private static FileSystem fs;
-
+
@Before
public void setup() throws ExecException, IOException {
pigServer = new PigServer(ExecType.LOCAL);
@@ -82,7 +97,7 @@ public class TestOrcStorage {
OUTPUT4 = OUTPUT4.replace("\\", "/");
}
}
-
+
@After
public void teardown() throws IOException {
if(pigServer != null) {
@@ -90,12 +105,12 @@ public class TestOrcStorage {
}
deleteTestFiles();
}
-
+
private void generateInputFiles() throws IOException {
String[] input = {"65536\tworld", "1\thello"};
Util.createLocalInputFile(INPUT1, input);
}
-
+
private static void deleteTestFiles() throws IOException {
Util.deleteDirectory(new File(outbasedir));
}
@@ -109,30 +124,10 @@ public class TestOrcStorage {
"middle: (list: {(int1: int,string1: chararray)}),list: {(int1: int,string1: chararray)}," +
"map: map[(int1: int,string1: chararray)],ts: datetime,decimal1: bigdecimal}");
Iterator<Tuple> iter = pigServer.openIterator("A");
- int count=0;
- Tuple t;
- while (iter.hasNext()) {
- t = (Tuple)iter.next();
- assertEquals(t.size(), 14);
- assertTrue(t.get(0) instanceof Boolean);
- assertTrue(t.get(1) instanceof Integer);
- assertTrue(t.get(2) instanceof Integer);
- assertTrue(t.get(3) instanceof Integer);
- assertTrue(t.get(4) instanceof Long);
- assertTrue(t.get(5) instanceof Float);
- assertTrue(t.get(6) instanceof Double);
- assertTrue(t.get(7) instanceof DataByteArray);
- assertTrue(t.get(8) instanceof String);
- assertTrue(t.get(9) instanceof Tuple);
- assertTrue(t.get(10) instanceof DataBag);
- assertTrue(t.get(11) instanceof Map);
- assertTrue(t.get(12) instanceof DateTime);
- assertTrue(t.get(13) instanceof BigDecimal);
- count++;
- }
- assertEquals(count, 7500);
+
+ verifyData(new Path(basedir + "orc-file-11-format.orc"), iter, fs, 7500);
}
-
+
@Test
public void testJoinWithPruning() throws Exception {
pigServer.registerQuery("A = load '" + basedir + "orc-file-11-format.orc'" + " using OrcStorage();" );
@@ -159,7 +154,7 @@ public class TestOrcStorage {
Path outputFilePath = new Path(new Path(OUTPUT1), "part-m-00000");
Reader reader = OrcFile.createReader(fs, outputFilePath);
assertEquals(reader.getNumberOfRows(), 2);
-
+
RecordReader rows = reader.rows(null);
Object row = rows.next(null);
StructObjectInspector soi = (StructObjectInspector)reader.getObjectInspector();
@@ -169,7 +164,7 @@ public class TestOrcStorage {
soi.getAllStructFieldRefs().get(1));
assertEquals(intWritable.get(), 65536);
assertEquals(text.toString(), "world");
-
+
row = rows.next(null);
intWritable = (IntWritable)soi.getStructFieldData(row,
soi.getAllStructFieldRefs().get(0));
@@ -177,10 +172,10 @@ public class TestOrcStorage {
soi.getAllStructFieldRefs().get(1));
assertEquals(intWritable.get(), 1);
assertEquals(text.toString(), "hello");
-
+
// A bug in ORC InputFormat does not allow empty file in input directory
- fs.delete(new Path(OUTPUT1, "_SUCCESS"));
-
+ fs.delete(new Path(OUTPUT1, "_SUCCESS"), true);
+
// Read the output file back
pigServer.registerQuery("A = load '" + OUTPUT1 + "' using OrcStorage();");
Schema s = pigServer.dumpSchema("A");
@@ -190,12 +185,12 @@ public class TestOrcStorage {
assertEquals(t.size(), 2);
assertEquals(t.get(0), 65536);
assertEquals(t.get(1), "world");
-
+
t = iter.next();
assertEquals(t.size(), 2);
assertEquals(t.get(0), 1);
assertEquals(t.get(1), "hello");
-
+
assertFalse(iter.hasNext());
}
@@ -207,35 +202,168 @@ public class TestOrcStorage {
pigServer.registerQuery("store B into '" + OUTPUT2 + "' using OrcStorage();");
pigServer.registerQuery("store B into '" + OUTPUT3 +"' using OrcStorage('-c SNAPPY');");
pigServer.executeBatch();
-
+
Path outputFilePath = new Path(new Path(OUTPUT2), "part-r-00000");
Reader reader = OrcFile.createReader(fs, outputFilePath);
assertEquals(reader.getNumberOfRows(), 2);
assertEquals(reader.getCompression(), CompressionKind.ZLIB);
-
- outputFilePath = new Path(new Path(OUTPUT3), "part-r-00000");
- reader = OrcFile.createReader(fs, outputFilePath);
+
+ Path outputFilePath2 = new Path(new Path(OUTPUT3), "part-r-00000");
+ reader = OrcFile.createReader(fs, outputFilePath2);
assertEquals(reader.getNumberOfRows(), 2);
assertEquals(reader.getCompression(), CompressionKind.SNAPPY);
+
+ verifyData(outputFilePath, outputFilePath2, fs, 2);
}
-
-
+
+ @Test
+ public void testMultipleLoadStore() throws Exception {
+ pigServer.registerQuery("A = load '" + basedir + "orc-file-11-format.orc'" + " using OrcStorage();" );
+ pigServer.registerQuery("store A into '" + OUTPUT1 + "' using OrcStorage();");
+ pigServer.registerQuery("B = load '" + OUTPUT1 + "' using OrcStorage();");
+ verifyData(new Path(basedir + "orc-file-11-format.orc"), pigServer.openIterator("B"), fs, 7500);
+ }
+
@Test
public void testLoadStoreMoreDataType() throws Exception {
pigServer.registerQuery("A = load '" + basedir + "orc-file-11-format.orc'" + " using OrcStorage();" );
pigServer.registerQuery("B = foreach A generate boolean1..double1, '' as bytes1, string1..;");
pigServer.store("B", OUTPUT4, "OrcStorage");
-
+
// A bug in ORC InputFormat does not allow empty file in input directory
- fs.delete(new Path(OUTPUT4, "_SUCCESS"));
-
+ fs.delete(new Path(OUTPUT4, "_SUCCESS"), true);
+
pigServer.registerQuery("A = load '" + OUTPUT4 + "' using OrcStorage();" );
Schema s = pigServer.dumpSchema("A");
Iterator<Tuple> iter = pigServer.openIterator("A");
Tuple t = iter.next();
assertTrue(t.toString().startsWith("(false,1,1024,65536,9223372036854775807,1.0,-15.0," +
",hi,({(1,bye),(2,sigh)}),{(3,good),(4,bad)},[],"));
- assertTrue(t.get(12).toString().matches("2000-03-12T00:00:00.000.*"));
+ assertTrue(t.get(12).toString().matches("2000-03-12T15:00:00.000.*"));
assertTrue(t.toString().endsWith(",12345678.6547456)"));
}
+
+ private void verifyData(Path orcFile, Iterator<Tuple> iter, FileSystem fs, int expectedTotalRows) throws Exception {
+
+ int expectedRows = 0;
+ int actualRows = 0;
+ Reader orcReader = OrcFile.createReader(fs, orcFile);
+ ObjectInspector oi = orcReader.getObjectInspector();
+ StructObjectInspector soi = (StructObjectInspector) oi;
+
+ RecordReader reader = orcReader.rows(null);
+ Object row = null;
+
+ while (reader.hasNext()) {
+ row = reader.next(row);
+ expectedRows++;
+ List<?> orcRow = soi.getStructFieldsDataAsList(row);
+ if (!iter.hasNext()) {
+ break;
+ }
+ Tuple t = iter.next();
+ assertEquals(orcRow.size(), t.size());
+ actualRows++;
+
+ for (int i = 0; i < orcRow.size(); i++) {
+ Object expected = orcRow.get(i);
+ Object actual = t.get(i);
+ compareData(expected, actual);
+ }
+ }
+ assertFalse(iter.hasNext());
+ assertEquals(expectedRows, actualRows);
+ assertEquals(expectedTotalRows, actualRows);
+
+ }
+
+ private void verifyData(Path orcFile, Path pigOrcFile, FileSystem fs, int expectedTotalRows) throws Exception {
+
+ int expectedRows = 0;
+ int actualRows = 0;
+ Reader orcReaderExpected = OrcFile.createReader(fs, orcFile);
+ StructObjectInspector soiExpected = (StructObjectInspector) orcReaderExpected.getObjectInspector();
+ Reader orcReaderActual = OrcFile.createReader(fs, orcFile);
+ StructObjectInspector soiActual = (StructObjectInspector) orcReaderActual.getObjectInspector();
+
+ RecordReader readerExpected = orcReaderExpected.rows(null);
+ Object expectedRow = null;
+ RecordReader readerActual = orcReaderActual.rows(null);
+ Object actualRow = null;
+
+ while (readerExpected.hasNext()) {
+ expectedRow = readerExpected.next(expectedRow);
+ expectedRows++;
+ List<?> orcRowExpected = soiExpected.getStructFieldsDataAsList(expectedRow);
+ if (!readerActual.hasNext()) {
+ break;
+ }
+ actualRow = readerActual.next(actualRow);
+ actualRows++;
+ List<?> orcRowActual = soiActual.getStructFieldsDataAsList(actualRow);
+ assertEquals(orcRowExpected.size(), orcRowActual.size());
+
+ for (int i = 0; i < orcRowExpected.size(); i++) {
+ assertEquals(orcRowExpected.get(i), orcRowActual.get(i));
+ }
+ }
+ assertFalse(readerActual.hasNext());
+ assertEquals(expectedRows, actualRows);
+ assertEquals(expectedTotalRows, actualRows);
+
+ }
+
+ private void compareData(Object expected, Object actual) {
+ if (expected instanceof Text) {
+ assertEquals(String.class, actual.getClass());
+ assertEquals(expected.toString(), actual);
+ } else if (expected instanceof ShortWritable) {
+ assertEquals(Integer.class, actual.getClass());
+ assertEquals((int)((ShortWritable) expected).get(), actual);
+ } else if (expected instanceof IntWritable) {
+ assertEquals(Integer.class, actual.getClass());
+ assertEquals(((IntWritable) expected).get(), actual);
+ } else if (expected instanceof LongWritable) {
+ assertEquals(Long.class, actual.getClass());
+ assertEquals(((LongWritable) expected).get(), actual);
+ } else if (expected instanceof FloatWritable) {
+ assertEquals(Float.class, actual.getClass());
+ assertEquals(((FloatWritable) expected).get(), actual);
+ } else if (expected instanceof HiveDecimalWritable) {
+ assertEquals(BigDecimal.class, actual.getClass());
+ assertEquals(((HiveDecimalWritable) expected).toString(), actual.toString());
+ } else if (expected instanceof DoubleWritable) {
+ assertEquals(Double.class, actual.getClass());
+ assertEquals(((DoubleWritable) expected).get(), actual);
+ } else if (expected instanceof BooleanWritable) {
+ assertEquals(Boolean.class, actual.getClass());
+ assertEquals(((BooleanWritable) expected).get(), actual);
+ } else if (expected instanceof TimestampWritable) {
+ assertEquals(DateTime.class, actual.getClass());
+ assertEquals(((TimestampWritable) expected).getTimestamp().getTime(),
+ ((DateTime) actual).getMillis());
+ } else if (expected instanceof BytesWritable) {
+ assertEquals(DataByteArray.class, actual.getClass());
+ BytesWritable bw = (BytesWritable) expected;
+ assertEquals(new DataByteArray(bw.getBytes(), 0, bw.getLength()), actual);
+ } else if (expected instanceof ByteWritable) {
+ assertEquals(Integer.class, actual.getClass());
+ assertEquals((int) ((ByteWritable) expected).get(), actual);
+ } else if (expected instanceof OrcStruct) {
+ assertEquals(BinSedesTuple.class, actual.getClass());
+ // TODO: compare actual values. No getters in OrcStruct
+ } else if (expected instanceof ArrayList) {
+ assertEquals(DefaultDataBag.class, actual.getClass());
+ // TODO: compare actual values. No getters in OrcStruct
+ } else if (expected instanceof HashMap) {
+ assertEquals(HashMap.class, actual.getClass());
+ assertEquals(((HashMap) expected).size(), ((HashMap) actual).size());
+ // TODO: compare actual values. No getters in OrcStruct
+ } else if (expected == null) {
+ assertEquals(expected, actual);
+ } else {
+ Assert.fail("Unknown object type: " + expected.getClass().getName());
+ }
+ }
+
}