You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2014/10/14 21:07:05 UTC
svn commit: r1631841 [41/42] - in /hive/branches/llap: ./ accumulo-handler/
accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/columns/
accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/mr/
accumulo-handler/src/java/org/apache/hadoop/hiv...
Modified: hive/branches/llap/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/SetIntString.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/SetIntString.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/SetIntString.java (original)
+++ hive/branches/llap/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde2/thrift/test/SetIntString.java Tue Oct 14 19:06:45 2014
@@ -6,6 +6,7 @@
*/
package org.apache.hadoop.hive.serde2.thrift.test;
+import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.thrift.scheme.IScheme;
import org.apache.thrift.scheme.SchemeFactory;
import org.apache.thrift.scheme.StandardScheme;
@@ -42,8 +43,8 @@ public class SetIntString implements org
schemes.put(TupleScheme.class, new SetIntStringTupleSchemeFactory());
}
- public Set<IntString> sIntString; // required
- public String aString; // required
+ private Set<IntString> sIntString; // required
+ private String aString; // required
/** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
public enum _Fields implements org.apache.thrift.TFieldIdEnum {
@@ -176,9 +177,8 @@ public class SetIntString implements org
return this.sIntString;
}
- public SetIntString setSIntString(Set<IntString> sIntString) {
+ public void setSIntString(Set<IntString> sIntString) {
this.sIntString = sIntString;
- return this;
}
public void unsetSIntString() {
@@ -200,9 +200,8 @@ public class SetIntString implements org
return this.aString;
}
- public SetIntString setAString(String aString) {
+ public void setAString(String aString) {
this.aString = aString;
- return this;
}
public void unsetAString() {
@@ -304,7 +303,19 @@ public class SetIntString implements org
@Override
public int hashCode() {
- return 0;
+ HashCodeBuilder builder = new HashCodeBuilder();
+
+ boolean present_sIntString = true && (isSetSIntString());
+ builder.append(present_sIntString);
+ if (present_sIntString)
+ builder.append(sIntString);
+
+ boolean present_aString = true && (isSetAString());
+ builder.append(present_aString);
+ if (present_aString)
+ builder.append(aString);
+
+ return builder.toHashCode();
}
public int compareTo(SetIntString other) {
@@ -416,14 +427,14 @@ public class SetIntString implements org
case 1: // S_INT_STRING
if (schemeField.type == org.apache.thrift.protocol.TType.SET) {
{
- org.apache.thrift.protocol.TSet _set34 = iprot.readSetBegin();
- struct.sIntString = new HashSet<IntString>(2*_set34.size);
- for (int _i35 = 0; _i35 < _set34.size; ++_i35)
+ org.apache.thrift.protocol.TSet _set82 = iprot.readSetBegin();
+ struct.sIntString = new HashSet<IntString>(2*_set82.size);
+ for (int _i83 = 0; _i83 < _set82.size; ++_i83)
{
- IntString _elem36; // required
- _elem36 = new IntString();
- _elem36.read(iprot);
- struct.sIntString.add(_elem36);
+ IntString _elem84; // required
+ _elem84 = new IntString();
+ _elem84.read(iprot);
+ struct.sIntString.add(_elem84);
}
iprot.readSetEnd();
}
@@ -446,8 +457,6 @@ public class SetIntString implements org
iprot.readFieldEnd();
}
iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
struct.validate();
}
@@ -459,9 +468,9 @@ public class SetIntString implements org
oprot.writeFieldBegin(S_INT_STRING_FIELD_DESC);
{
oprot.writeSetBegin(new org.apache.thrift.protocol.TSet(org.apache.thrift.protocol.TType.STRUCT, struct.sIntString.size()));
- for (IntString _iter37 : struct.sIntString)
+ for (IntString _iter85 : struct.sIntString)
{
- _iter37.write(oprot);
+ _iter85.write(oprot);
}
oprot.writeSetEnd();
}
@@ -500,9 +509,9 @@ public class SetIntString implements org
if (struct.isSetSIntString()) {
{
oprot.writeI32(struct.sIntString.size());
- for (IntString _iter38 : struct.sIntString)
+ for (IntString _iter86 : struct.sIntString)
{
- _iter38.write(oprot);
+ _iter86.write(oprot);
}
}
}
@@ -517,14 +526,14 @@ public class SetIntString implements org
BitSet incoming = iprot.readBitSet(2);
if (incoming.get(0)) {
{
- org.apache.thrift.protocol.TSet _set39 = new org.apache.thrift.protocol.TSet(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- struct.sIntString = new HashSet<IntString>(2*_set39.size);
- for (int _i40 = 0; _i40 < _set39.size; ++_i40)
+ org.apache.thrift.protocol.TSet _set87 = new org.apache.thrift.protocol.TSet(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+ struct.sIntString = new HashSet<IntString>(2*_set87.size);
+ for (int _i88 = 0; _i88 < _set87.size; ++_i88)
{
- IntString _elem41; // required
- _elem41 = new IntString();
- _elem41.read(iprot);
- struct.sIntString.add(_elem41);
+ IntString _elem89; // required
+ _elem89 = new IntString();
+ _elem89.read(iprot);
+ struct.sIntString.add(_elem89);
}
}
struct.setSIntStringIsSet(true);
Modified: hive/branches/llap/serde/src/gen/thrift/gen-py/complex/ttypes.py
URL: http://svn.apache.org/viewvc/hive/branches/llap/serde/src/gen/thrift/gen-py/complex/ttypes.py?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/serde/src/gen/thrift/gen-py/complex/ttypes.py (original)
+++ hive/branches/llap/serde/src/gen/thrift/gen-py/complex/ttypes.py Tue Oct 14 19:06:45 2014
@@ -487,3 +487,84 @@ class Complex:
def __ne__(self, other):
return not (self == other)
+
+class SetIntString:
+ """
+ Attributes:
+ - sIntString
+ - aString
+ """
+
+ thrift_spec = (
+ None, # 0
+ (1, TType.SET, 'sIntString', (TType.STRUCT,(IntString, IntString.thrift_spec)), None, ), # 1
+ (2, TType.STRING, 'aString', None, None, ), # 2
+ )
+
+ def __init__(self, sIntString=None, aString=None,):
+ self.sIntString = sIntString
+ self.aString = aString
+
+ def read(self, iprot):
+ if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+ fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+ return
+ iprot.readStructBegin()
+ while True:
+ (fname, ftype, fid) = iprot.readFieldBegin()
+ if ftype == TType.STOP:
+ break
+ if fid == 1:
+ if ftype == TType.SET:
+ self.sIntString = set()
+ (_etype76, _size73) = iprot.readSetBegin()
+ for _i77 in xrange(_size73):
+ _elem78 = IntString()
+ _elem78.read(iprot)
+ self.sIntString.add(_elem78)
+ iprot.readSetEnd()
+ else:
+ iprot.skip(ftype)
+ elif fid == 2:
+ if ftype == TType.STRING:
+ self.aString = iprot.readString();
+ else:
+ iprot.skip(ftype)
+ else:
+ iprot.skip(ftype)
+ iprot.readFieldEnd()
+ iprot.readStructEnd()
+
+ def write(self, oprot):
+ if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+ oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+ return
+ oprot.writeStructBegin('SetIntString')
+ if self.sIntString is not None:
+ oprot.writeFieldBegin('sIntString', TType.SET, 1)
+ oprot.writeSetBegin(TType.STRUCT, len(self.sIntString))
+ for iter79 in self.sIntString:
+ iter79.write(oprot)
+ oprot.writeSetEnd()
+ oprot.writeFieldEnd()
+ if self.aString is not None:
+ oprot.writeFieldBegin('aString', TType.STRING, 2)
+ oprot.writeString(self.aString)
+ oprot.writeFieldEnd()
+ oprot.writeFieldStop()
+ oprot.writeStructEnd()
+
+ def validate(self):
+ return
+
+
+ def __repr__(self):
+ L = ['%s=%r' % (key, value)
+ for key, value in self.__dict__.iteritems()]
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+ def __eq__(self, other):
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+ def __ne__(self, other):
+ return not (self == other)
Modified: hive/branches/llap/serde/src/gen/thrift/gen-rb/complex_types.rb
URL: http://svn.apache.org/viewvc/hive/branches/llap/serde/src/gen/thrift/gen-rb/complex_types.rb?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/serde/src/gen/thrift/gen-rb/complex_types.rb (original)
+++ hive/branches/llap/serde/src/gen/thrift/gen-rb/complex_types.rb Tue Oct 14 19:06:45 2014
@@ -119,3 +119,21 @@ class Complex
::Thrift::Struct.generate_accessors self
end
+class SetIntString
+ include ::Thrift::Struct, ::Thrift::Struct_Union
+ SINTSTRING = 1
+ ASTRING = 2
+
+ FIELDS = {
+ SINTSTRING => {:type => ::Thrift::Types::SET, :name => 'sIntString', :element => {:type => ::Thrift::Types::STRUCT, :class => ::IntString}},
+ ASTRING => {:type => ::Thrift::Types::STRING, :name => 'aString'}
+ }
+
+ def struct_fields; FIELDS; end
+
+ def validate
+ end
+
+ ::Thrift::Struct.generate_accessors self
+end
+
Modified: hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java (original)
+++ hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java Tue Oct 14 19:06:45 2014
@@ -22,6 +22,7 @@ import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.rmi.server.UID;
+import java.sql.Date;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -39,10 +40,12 @@ import org.apache.avro.io.BinaryDecoder;
import org.apache.avro.io.BinaryEncoder;
import org.apache.avro.io.DecoderFactory;
import org.apache.avro.io.EncoderFactory;
-import org.apache.avro.util.Utf8;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.common.type.HiveChar;
import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
import org.apache.hadoop.hive.serde2.objectinspector.StandardUnionObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaHiveDecimalObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
@@ -200,7 +203,6 @@ class AvroDeserializer {
return deserializeNullableUnion(datum, fileSchema, recordSchema, columnType);
}
-
switch(columnType.getCategory()) {
case STRUCT:
return deserializeStruct((GenericData.Record) datum, fileSchema, (StructTypeInfo) columnType);
@@ -249,6 +251,42 @@ class AvroDeserializer {
JavaHiveDecimalObjectInspector oi = (JavaHiveDecimalObjectInspector)
PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector((DecimalTypeInfo)columnType);
return oi.set(null, dec);
+ case CHAR:
+ if (fileSchema == null) {
+ throw new AvroSerdeException("File schema is missing for char field. Reader schema is " + columnType);
+ }
+
+ int maxLength = 0;
+ try {
+ maxLength = fileSchema.getJsonProp(AvroSerDe.AVRO_PROP_MAX_LENGTH).getValueAsInt();
+ } catch (Exception ex) {
+ throw new AvroSerdeException("Failed to obtain maxLength value for char field from file schema: " + fileSchema, ex);
+ }
+
+ String str = datum.toString();
+ HiveChar hc = new HiveChar(str, maxLength);
+ return hc;
+ case VARCHAR:
+ if (fileSchema == null) {
+ throw new AvroSerdeException("File schema is missing for varchar field. Reader schema is " + columnType);
+ }
+
+ maxLength = 0;
+ try {
+ maxLength = fileSchema.getJsonProp(AvroSerDe.AVRO_PROP_MAX_LENGTH).getValueAsInt();
+ } catch (Exception ex) {
+ throw new AvroSerdeException("Failed to obtain maxLength value for varchar field from file schema: " + fileSchema, ex);
+ }
+
+ str = datum.toString();
+ HiveVarchar hvc = new HiveVarchar(str, maxLength);
+ return hvc;
+ case DATE:
+ if (recordSchema.getType() != Type.INT) {
+ throw new AvroSerdeException("Unexpected Avro schema for Date TypeInfo: " + recordSchema.getType());
+ }
+
+ return new Date(DateWritable.daysToMillis((Integer)datum));
default:
return datum;
}
@@ -331,10 +369,10 @@ class AvroDeserializer {
// Avro only allows maps with Strings for keys, so we only have to worry
// about deserializing the values
Map<String, Object> map = new HashMap<String, Object>();
- Map<Utf8, Object> mapDatum = (Map)datum;
+ Map<CharSequence, Object> mapDatum = (Map)datum;
Schema valueSchema = mapSchema.getValueType();
TypeInfo valueTypeInfo = columnType.getMapValueTypeInfo();
- for (Utf8 key : mapDatum.keySet()) {
+ for (CharSequence key : mapDatum.keySet()) {
Object value = mapDatum.get(key);
map.put(key.toString(), worker(value, fileSchema == null ? null : fileSchema.getValueType(),
valueSchema, valueTypeInfo));
Modified: hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java (original)
+++ hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java Tue Oct 14 19:06:45 2014
@@ -41,9 +41,15 @@ public class AvroSerDe extends AbstractS
private static final Log LOG = LogFactory.getLog(AvroSerDe.class);
public static final String DECIMAL_TYPE_NAME = "decimal";
+ public static final String CHAR_TYPE_NAME = "char";
+ public static final String VARCHAR_TYPE_NAME = "varchar";
+ public static final String DATE_TYPE_NAME = "date";
public static final String AVRO_PROP_LOGICAL_TYPE = "logicalType";
public static final String AVRO_PROP_PRECISION = "precision";
public static final String AVRO_PROP_SCALE = "scale";
+ public static final String AVRO_PROP_MAX_LENGTH = "maxLength";
+ public static final String AVRO_STRING_TYPE_NAME = "string";
+ public static final String AVRO_INT_TYPE_NAME = "int";
private ObjectInspector oi;
private List<String> columnNames;
Modified: hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerializer.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerializer.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerializer.java (original)
+++ hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerializer.java Tue Oct 14 19:06:45 2014
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hive.serde2.avro;
+import java.sql.Date;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -30,7 +31,10 @@ import org.apache.avro.generic.GenericDa
import org.apache.avro.generic.GenericEnumSymbol;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.common.type.HiveChar;
import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -38,6 +42,7 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
@@ -193,6 +198,15 @@ class AvroSerializer {
case DECIMAL:
HiveDecimal dec = (HiveDecimal)fieldOI.getPrimitiveJavaObject(structFieldData);
return AvroSerdeUtils.getBufferFromDecimal(dec, ((DecimalTypeInfo)typeInfo).scale());
+ case CHAR:
+ HiveChar ch = (HiveChar)fieldOI.getPrimitiveJavaObject(structFieldData);
+ return ch.getStrippedValue();
+ case VARCHAR:
+ HiveVarchar vc = (HiveVarchar)fieldOI.getPrimitiveJavaObject(structFieldData);
+ return vc.getValue();
+ case DATE:
+ Date date = ((DateObjectInspector)fieldOI).getPrimitiveJavaObject(structFieldData);
+ return DateWritable.dateToDays(date);
case UNKNOWN:
throw new AvroSerdeException("Received UNKNOWN primitive category.");
case VOID:
Modified: hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java (original)
+++ hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java Tue Oct 14 19:06:45 2014
@@ -128,6 +128,33 @@ class SchemaToTypeInfo {
return TypeInfoFactory.getDecimalTypeInfo(precision, scale);
}
+ if (type == Schema.Type.STRING &&
+ AvroSerDe.CHAR_TYPE_NAME.equalsIgnoreCase(schema.getProp(AvroSerDe.AVRO_PROP_LOGICAL_TYPE))) {
+ int maxLength = 0;
+ try {
+ maxLength = schema.getJsonProp(AvroSerDe.AVRO_PROP_MAX_LENGTH).getValueAsInt();
+ } catch (Exception ex) {
+ throw new AvroSerdeException("Failed to obtain maxLength value from file schema: " + schema, ex);
+ }
+ return TypeInfoFactory.getCharTypeInfo(maxLength);
+ }
+
+ if (type == Schema.Type.STRING &&
+ AvroSerDe.VARCHAR_TYPE_NAME.equalsIgnoreCase(schema.getProp(AvroSerDe.AVRO_PROP_LOGICAL_TYPE))) {
+ int maxLength = 0;
+ try {
+ maxLength = schema.getJsonProp(AvroSerDe.AVRO_PROP_MAX_LENGTH).getValueAsInt();
+ } catch (Exception ex) {
+ throw new AvroSerdeException("Failed to obtain maxLength value from file schema: " + schema, ex);
+ }
+ return TypeInfoFactory.getVarcharTypeInfo(maxLength);
+ }
+
+ if (type == Schema.Type.INT &&
+ AvroSerDe.DATE_TYPE_NAME.equals(schema.getProp(AvroSerDe.AVRO_PROP_LOGICAL_TYPE))) {
+ return TypeInfoFactory.dateTypeInfo;
+ }
+
return typeInfoCache.retrieve(schema);
}
Modified: hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/avro/TypeInfoToSchema.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/avro/TypeInfoToSchema.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/avro/TypeInfoToSchema.java (original)
+++ hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/avro/TypeInfoToSchema.java Tue Oct 14 19:06:45 2014
@@ -19,6 +19,7 @@ package org.apache.hadoop.hive.serde2.av
import org.apache.avro.Schema;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
@@ -26,6 +27,7 @@ import org.apache.hadoop.hive.serde2.typ
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo;
import org.codehaus.jackson.JsonNode;
import org.codehaus.jackson.node.JsonNodeFactory;
@@ -105,10 +107,16 @@ public class TypeInfoToSchema {
schema = Schema.create(Schema.Type.STRING);
break;
case CHAR:
- schema = Schema.create(Schema.Type.STRING);
+ schema = AvroSerdeUtils.getSchemaFor("{" +
+ "\"type\":\"" + AvroSerDe.AVRO_STRING_TYPE_NAME + "\"," +
+ "\"logicalType\":\"" + AvroSerDe.CHAR_TYPE_NAME + "\"," +
+ "\"maxLength\":" + ((CharTypeInfo) typeInfo).getLength() + "}");
break;
case VARCHAR:
- schema = Schema.create(Schema.Type.STRING);
+ schema = AvroSerdeUtils.getSchemaFor("{" +
+ "\"type\":\"" + AvroSerDe.AVRO_STRING_TYPE_NAME + "\"," +
+ "\"logicalType\":\"" + AvroSerDe.VARCHAR_TYPE_NAME + "\"," +
+ "\"maxLength\":" + ((VarcharTypeInfo) typeInfo).getLength() + "}");
break;
case BINARY:
schema = Schema.create(Schema.Type.BYTES);
@@ -144,6 +152,11 @@ public class TypeInfoToSchema {
"\"precision\":" + precision + "," +
"\"scale\":" + scale + "}");
break;
+ case DATE:
+ schema = AvroSerdeUtils.getSchemaFor("{" +
+ "\"type\":\"" + AvroSerDe.AVRO_INT_TYPE_NAME + "\"," +
+ "\"logicalType\":\"" + AvroSerDe.DATE_TYPE_NAME + "\"}");
+ break;
case VOID:
schema = Schema.create(Schema.Type.NULL);
break;
Modified: hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveChar.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveChar.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveChar.java (original)
+++ hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveChar.java Tue Oct 14 19:06:45 2014
@@ -55,14 +55,24 @@ public class LazyHiveChar extends
@Override
public void init(ByteArrayRef bytes, int start, int length) {
- String byteData = null;
- try {
- byteData = Text.decode(bytes.getData(), start, length);
- data.set(byteData, maxLength);
+ if (oi.isEscaped()) {
+ Text textData = data.getTextValue();
+ // This is doing a lot of copying here, this could be improved by enforcing length
+ // at the same time as escaping rather than as separate steps.
+ LazyUtils.copyAndEscapeStringDataToText(bytes.getData(), start, length,
+ oi.getEscapeChar(),textData);
+ data.set(textData.toString(), maxLength);
isNull = false;
- } catch (CharacterCodingException e) {
- isNull = true;
- LOG.debug("Data not in the HiveChar data type range so converted to null.", e);
+ } else {
+ String byteData = null;
+ try {
+ byteData = Text.decode(bytes.getData(), start, length);
+ data.set(byteData, maxLength);
+ isNull = false;
+ } catch (CharacterCodingException e) {
+ isNull = true;
+ LOG.debug("Data not in the HiveChar data type range so converted to null.", e);
+ }
}
}
Modified: hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveVarchar.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveVarchar.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveVarchar.java (original)
+++ hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveVarchar.java Tue Oct 14 19:06:45 2014
@@ -55,14 +55,24 @@ public class LazyHiveVarchar extends
@Override
public void init(ByteArrayRef bytes, int start, int length) {
- String byteData = null;
- try {
- byteData = Text.decode(bytes.getData(), start, length);
- data.set(byteData, maxLength);
+ if (oi.isEscaped()) {
+ Text textData = data.getTextValue();
+ // This is doing a lot of copying here, this could be improved by enforcing length
+ // at the same time as escaping rather than as separate steps.
+ LazyUtils.copyAndEscapeStringDataToText(bytes.getData(), start, length,
+ oi.getEscapeChar(),textData);
+ data.set(textData.toString(), maxLength);
isNull = false;
- } catch (CharacterCodingException e) {
- isNull = true;
- LOG.debug("Data not in the HiveVarchar data type range so converted to null.", e);
+ } else {
+ try {
+ String byteData = null;
+ byteData = Text.decode(bytes.getData(), start, length);
+ data.set(byteData, maxLength);
+ isNull = false;
+ } catch (CharacterCodingException e) {
+ isNull = true;
+ LOG.debug("Data not in the HiveVarchar data type range so converted to null.", e);
+ }
}
}
Modified: hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java (original)
+++ hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java Tue Oct 14 19:06:45 2014
@@ -23,6 +23,8 @@ import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyMapObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.io.Text;
@@ -36,6 +38,7 @@ import org.apache.hadoop.io.Text;
*/
public class LazyMap extends LazyNonPrimitive<LazyMapObjectInspector> {
+ public static final Log LOG = LogFactory.getLog(LazyMap.class);
/**
* Whether the data is already parsed or not.
*/
@@ -170,15 +173,19 @@ public class LazyMap extends LazyNonPrim
valueLength[mapSize] = elementByteEnd - (keyEnd[mapSize] + 1);
LazyPrimitive<?, ?> lazyKey = uncheckedGetKey(mapSize);
if (lazyKey == null) {
- continue;
- }
- Object key = lazyKey.getObject();
- if(!keySet.contains(key)) {
- mapSize++;
- keySet.add(key);
- } else {
+ LOG.warn("skipped empty entry or entry with empty key in the representation of column with MAP type.");
+ //reset keyInited[mapSize] flag, since it may be set to true in the case of previous empty entry
keyInited[mapSize] = false;
+ } else {
+ Object key = lazyKey.getObject();
+ if(!keySet.contains(key)) {
+ mapSize++;
+ keySet.add(key);
+ } else {
+ keyInited[mapSize] = false;
+ }
}
+
// reset keyValueSeparatorPosition
keyValueSeparatorPosition = -1;
elementByteBegin = elementByteEnd + 1;
Modified: hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java (original)
+++ hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java Tue Oct 14 19:06:45 2014
@@ -40,40 +40,7 @@ public class LazyString extends LazyPrim
if (oi.isEscaped()) {
byte escapeChar = oi.getEscapeChar();
byte[] inputBytes = bytes.getData();
-
- // First calculate the length of the output string
- int outputLength = 0;
- for (int i = 0; i < length; i++) {
- if (inputBytes[start + i] != escapeChar) {
- outputLength++;
- } else {
- outputLength++;
- i++;
- }
- }
-
- // Copy the data over, so that the internal state of Text will be set to
- // the required outputLength.
- data.set(bytes.getData(), start, outputLength);
-
- // We need to copy the data byte by byte only in case the
- // "outputLength < length" (which means there is at least one escaped
- // byte.
- if (outputLength < length) {
- int k = 0;
- byte[] outputBytes = data.getBytes();
- for (int i = 0; i < length; i++) {
- byte b = inputBytes[start + i];
- if (b != escapeChar || i == length - 1) {
- outputBytes[k++] = b;
- } else {
- // get the next byte
- i++;
- outputBytes[k++] = inputBytes[start + i];
- }
- }
- assert (k == outputLength);
- }
+ LazyUtils.copyAndEscapeStringDataToText(inputBytes, start, length, escapeChar, data);
} else {
// if the data is not escaped, simply copy the data.
data.set(bytes.getData(), start, length);
Modified: hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java (original)
+++ hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java Tue Oct 14 19:06:45 2014
@@ -412,6 +412,44 @@ public final class LazyUtils {
}
}
+ public static void copyAndEscapeStringDataToText(byte[] inputBytes, int start, int length,
+ byte escapeChar, Text data) {
+
+ // First calculate the length of the output string
+ int outputLength = 0;
+ for (int i = 0; i < length; i++) {
+ if (inputBytes[start + i] != escapeChar) {
+ outputLength++;
+ } else {
+ outputLength++;
+ i++;
+ }
+ }
+
+ // Copy the data over, so that the internal state of Text will be set to
+ // the required outputLength.
+ data.set(inputBytes, start, outputLength);
+
+ // We need to copy the data byte by byte only in case the
+ // "outputLength < length" (which means there is at least one escaped
+ // byte.
+ if (outputLength < length) {
+ int k = 0;
+ byte[] outputBytes = data.getBytes();
+ for (int i = 0; i < length; i++) {
+ byte b = inputBytes[start + i];
+ if (b != escapeChar || i == length - 1) {
+ outputBytes[k++] = b;
+ } else {
+ // get the next byte
+ i++;
+ outputBytes[k++] = inputBytes[start + i];
+ }
+ }
+ assert (k == outputLength);
+ }
+ }
+
private LazyUtils() {
// prevent instantiation
}
Modified: hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveCharObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveCharObjectInspector.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveCharObjectInspector.java (original)
+++ hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveCharObjectInspector.java Tue Oct 14 19:06:45 2014
@@ -29,12 +29,21 @@ public class LazyHiveCharObjectInspector
extends AbstractPrimitiveLazyObjectInspector<HiveCharWritable>
implements HiveCharObjectInspector {
+ private boolean escaped;
+ private byte escapeChar;
+
// no-arg ctor required for Kyro
public LazyHiveCharObjectInspector() {
}
public LazyHiveCharObjectInspector(CharTypeInfo typeInfo) {
+ this(typeInfo, false, (byte)0);
+ }
+
+ public LazyHiveCharObjectInspector(CharTypeInfo typeInfo, boolean escaped, byte escapeChar) {
super(typeInfo);
+ this.escaped = escaped;
+ this.escapeChar = escapeChar;
}
@Override
@@ -63,6 +72,14 @@ public class LazyHiveCharObjectInspector
return ret;
}
+ public boolean isEscaped() {
+ return escaped;
+ }
+
+ public byte getEscapeChar() {
+ return escapeChar;
+ }
+
@Override
public String toString() {
return getTypeName();
Modified: hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveVarcharObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveVarcharObjectInspector.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveVarcharObjectInspector.java (original)
+++ hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveVarcharObjectInspector.java Tue Oct 14 19:06:45 2014
@@ -29,12 +29,21 @@ public class LazyHiveVarcharObjectInspec
extends AbstractPrimitiveLazyObjectInspector<HiveVarcharWritable>
implements HiveVarcharObjectInspector {
+ private boolean escaped;
+ private byte escapeChar;
+
// no-arg ctor required for Kyro
public LazyHiveVarcharObjectInspector() {
}
public LazyHiveVarcharObjectInspector(VarcharTypeInfo typeInfo) {
+ this(typeInfo, false, (byte)0);
+ }
+
+ public LazyHiveVarcharObjectInspector(VarcharTypeInfo typeInfo, boolean escaped, byte escapeChar) {
super(typeInfo);
+ this.escaped = escaped;
+ this.escapeChar = escapeChar;
}
@Override
@@ -63,6 +72,14 @@ public class LazyHiveVarcharObjectInspec
return ret;
}
+ public boolean isEscaped() {
+ return escaped;
+ }
+
+ public byte getEscapeChar() {
+ return escapeChar;
+ }
+
@Override
public String toString() {
return getTypeName();
Modified: hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java (original)
+++ hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java Tue Oct 14 19:06:45 2014
@@ -78,8 +78,10 @@ public final class LazyPrimitiveObjectIn
// prevent instantiation
}
- private static HashMap<ArrayList<Object>, LazyStringObjectInspector> cachedLazyStringObjectInspector =
- new HashMap<ArrayList<Object>, LazyStringObjectInspector>();
+ // Lazy object inspectors for string/char/varchar will all be cached in the same map.
+ // Map key will be list of [typeInfo, isEscaped, escapeChar]
+ private static HashMap<ArrayList<Object>, AbstractPrimitiveLazyObjectInspector> cachedLazyStringTypeOIs =
+ new HashMap<ArrayList<Object>, AbstractPrimitiveLazyObjectInspector>();
private static Map<PrimitiveTypeInfo, AbstractPrimitiveLazyObjectInspector<?>>
cachedPrimitiveLazyObjectInspectors =
@@ -121,6 +123,10 @@ public final class LazyPrimitiveObjectIn
switch(primitiveCategory) {
case STRING:
return getLazyStringObjectInspector(escaped, escapeChar);
+ case CHAR:
+ return getLazyHiveCharObjectInspector((CharTypeInfo)typeInfo, escaped, escapeChar);
+ case VARCHAR:
+ return getLazyHiveVarcharObjectInspector((VarcharTypeInfo)typeInfo, escaped, escapeChar);
case BOOLEAN:
return getLazyBooleanObjectInspector(extBoolean);
default:
@@ -157,13 +163,44 @@ public final class LazyPrimitiveObjectIn
public static LazyStringObjectInspector getLazyStringObjectInspector(boolean escaped, byte escapeChar) {
ArrayList<Object> signature = new ArrayList<Object>();
+ signature.add(TypeInfoFactory.stringTypeInfo);
signature.add(Boolean.valueOf(escaped));
signature.add(Byte.valueOf(escapeChar));
- LazyStringObjectInspector result = cachedLazyStringObjectInspector
+ LazyStringObjectInspector result = (LazyStringObjectInspector) cachedLazyStringTypeOIs
.get(signature);
if (result == null) {
result = new LazyStringObjectInspector(escaped, escapeChar);
- cachedLazyStringObjectInspector.put(signature, result);
+ cachedLazyStringTypeOIs.put(signature, result);
+ }
+ return result;
+ }
+
+ public static LazyHiveCharObjectInspector getLazyHiveCharObjectInspector(
+ CharTypeInfo typeInfo, boolean escaped, byte escapeChar) {
+ ArrayList<Object> signature = new ArrayList<Object>();
+ signature.add(typeInfo);
+ signature.add(Boolean.valueOf(escaped));
+ signature.add(Byte.valueOf(escapeChar));
+ LazyHiveCharObjectInspector result = (LazyHiveCharObjectInspector) cachedLazyStringTypeOIs
+ .get(signature);
+ if (result == null) {
+ result = new LazyHiveCharObjectInspector(typeInfo, escaped, escapeChar);
+ cachedLazyStringTypeOIs.put(signature, result);
+ }
+ return result;
+ }
+
+ public static LazyHiveVarcharObjectInspector getLazyHiveVarcharObjectInspector(
+ VarcharTypeInfo typeInfo, boolean escaped, byte escapeChar) {
+ ArrayList<Object> signature = new ArrayList<Object>();
+ signature.add(typeInfo);
+ signature.add(Boolean.valueOf(escaped));
+ signature.add(Byte.valueOf(escapeChar));
+ LazyHiveVarcharObjectInspector result = (LazyHiveVarcharObjectInspector) cachedLazyStringTypeOIs
+ .get(signature);
+ if (result == null) {
+ result = new LazyHiveVarcharObjectInspector(typeInfo, escaped, escapeChar);
+ cachedLazyStringTypeOIs.put(signature, result);
}
return result;
}
Modified: hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveCharObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveCharObjectInspector.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveCharObjectInspector.java (original)
+++ hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveCharObjectInspector.java Tue Oct 14 19:06:45 2014
@@ -21,6 +21,11 @@ import org.apache.hadoop.hive.common.typ
import org.apache.hadoop.hive.serde2.io.HiveCharWritable;
import org.apache.hadoop.hive.serde2.typeinfo.BaseCharUtils;
import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo;
+import org.apache.hadoop.io.Text;
+
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.Charset;
public class WritableHiveCharObjectInspector extends AbstractPrimitiveWritableObjectInspector
implements SettableHiveCharObjectInspector {
@@ -39,6 +44,12 @@ public class WritableHiveCharObjectInspe
if (o == null) {
return null;
}
+
+ if (o instanceof Text) {
+ String str = ((Text)o).toString();
+ return new HiveChar(str, ((CharTypeInfo)typeInfo).getLength());
+ }
+
HiveCharWritable writable = ((HiveCharWritable) o);
if (doesWritableMatchTypeParams(writable)) {
return writable.getHiveChar();
@@ -53,6 +64,14 @@ public class WritableHiveCharObjectInspe
if (o == null) {
return null;
}
+
+ if (o instanceof Text) {
+ String str = ((Text)o).toString();
+ HiveCharWritable hcw = new HiveCharWritable();
+ hcw.set(str, ((CharTypeInfo)typeInfo).getLength());
+ return hcw;
+ }
+
HiveCharWritable writable = ((HiveCharWritable) o);
if (doesWritableMatchTypeParams((HiveCharWritable) o)) {
return writable;
@@ -91,6 +110,9 @@ public class WritableHiveCharObjectInspe
@Override
public Object set(Object o, HiveChar value) {
+ if (value == null) {
+ return null;
+ }
HiveCharWritable writable = (HiveCharWritable) o;
writable.set(value, getMaxLength());
return o;
@@ -98,6 +120,9 @@ public class WritableHiveCharObjectInspe
@Override
public Object set(Object o, String value) {
+ if (value == null) {
+ return null;
+ }
HiveCharWritable writable = (HiveCharWritable) o;
writable.set(value, getMaxLength());
return o;
Modified: hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveVarcharObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveVarcharObjectInspector.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveVarcharObjectInspector.java (original)
+++ hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveVarcharObjectInspector.java Tue Oct 14 19:06:45 2014
@@ -19,10 +19,15 @@ package org.apache.hadoop.hive.serde2.ob
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.common.type.HiveChar;
import org.apache.hadoop.hive.common.type.HiveVarchar;
+import org.apache.hadoop.hive.serde2.io.HiveCharWritable;
import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable;
+import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.BaseCharUtils;
+import org.apache.hadoop.io.Text;
+import org.apache.hive.common.util.HiveStringUtils;
public class WritableHiveVarcharObjectInspector extends AbstractPrimitiveWritableObjectInspector
implements SettableHiveVarcharObjectInspector {
@@ -43,6 +48,12 @@ implements SettableHiveVarcharObjectInsp
if (o == null) {
return null;
}
+
+ if (o instanceof Text) {
+ String str = ((Text)o).toString();
+ return new HiveVarchar(str, ((VarcharTypeInfo)typeInfo).getLength());
+ }
+
HiveVarcharWritable writable = ((HiveVarcharWritable)o);
if (doesWritableMatchTypeParams(writable)) {
return writable.getHiveVarchar();
@@ -57,6 +68,14 @@ implements SettableHiveVarcharObjectInsp
if (o == null) {
return null;
}
+
+ if (o instanceof Text) {
+ String str = ((Text)o).toString();
+ HiveVarcharWritable hcw = new HiveVarcharWritable();
+ hcw.set(str, ((VarcharTypeInfo)typeInfo).getLength());
+ return hcw;
+ }
+
HiveVarcharWritable writable = ((HiveVarcharWritable)o);
if (doesWritableMatchTypeParams((HiveVarcharWritable)o)) {
return writable;
@@ -96,6 +115,9 @@ implements SettableHiveVarcharObjectInsp
@Override
public Object set(Object o, HiveVarchar value) {
+ if (value == null) {
+ return null;
+ }
HiveVarcharWritable writable = (HiveVarcharWritable)o;
writable.set(value, getMaxLength());
return o;
@@ -103,6 +125,9 @@ implements SettableHiveVarcharObjectInsp
@Override
public Object set(Object o, String value) {
+ if (value == null) {
+ return null;
+ }
HiveVarcharWritable writable = (HiveVarcharWritable)o;
writable.set(value, getMaxLength());
return o;
Modified: hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/BaseCharTypeInfo.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/BaseCharTypeInfo.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/BaseCharTypeInfo.java (original)
+++ hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/BaseCharTypeInfo.java Tue Oct 14 19:06:45 2014
@@ -27,6 +27,10 @@ public abstract class BaseCharTypeInfo e
public BaseCharTypeInfo() {
}
+ public BaseCharTypeInfo(String typeName) {
+ super(typeName);
+ }
+
public BaseCharTypeInfo(String typeName, int length) {
super(typeName);
this.length = length;
@@ -53,4 +57,9 @@ public abstract class BaseCharTypeInfo e
return sb.toString();
}
+ @Override
+ public void setTypeName(String typeName) {
+ // type name should already be set by subclass
+ return;
+ }
}
Modified: hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/CharTypeInfo.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/CharTypeInfo.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/CharTypeInfo.java (original)
+++ hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/CharTypeInfo.java Tue Oct 14 19:06:45 2014
@@ -25,6 +25,7 @@ public class CharTypeInfo extends BaseC
// no-arg constructor to make kyro happy.
public CharTypeInfo() {
+ super(serdeConstants.CHAR_TYPE_NAME);
}
public CharTypeInfo(int length) {
Modified: hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/DecimalTypeInfo.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/DecimalTypeInfo.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/DecimalTypeInfo.java (original)
+++ hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/DecimalTypeInfo.java Tue Oct 14 19:06:45 2014
@@ -28,6 +28,7 @@ public class DecimalTypeInfo extends Pri
// no-arg constructor to make kyro happy.
public DecimalTypeInfo() {
+ super(serdeConstants.DECIMAL_TYPE_NAME);
}
public DecimalTypeInfo(int precision, int scale) {
@@ -43,6 +44,12 @@ public class DecimalTypeInfo extends Pri
}
@Override
+ public void setTypeName(String typeName) {
+ // No need to set type name, it should always be decimal
+ return;
+ }
+
+ @Override
public boolean equals(Object other) {
if (other == null || !(other instanceof DecimalTypeInfo)) {
return false;
@@ -101,4 +108,20 @@ public class DecimalTypeInfo extends Pri
return this.precision() - this.scale() >= dti.precision() - dti.scale();
}
+ public int getPrecision() {
+ return precision;
+ }
+
+ public void setPrecision(int precision) {
+ this.precision = precision;
+ }
+
+ public int getScale() {
+ return scale;
+ }
+
+ public void setScale(int scale) {
+ this.scale = scale;
+ }
+
}
Modified: hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/HiveDecimalUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/HiveDecimalUtils.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/HiveDecimalUtils.java (original)
+++ hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/HiveDecimalUtils.java Tue Oct 14 19:06:45 2014
@@ -22,6 +22,7 @@ import java.math.BigDecimal;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
public class HiveDecimalUtils {
@@ -134,4 +135,25 @@ public class HiveDecimalUtils {
}
}
+ public static TypeInfo getDecimalTypeForPrimitiveCategories(
+ PrimitiveTypeInfo a, PrimitiveTypeInfo b) {
+ int prec1 = HiveDecimalUtils.getPrecisionForType(a);
+ int prec2 = HiveDecimalUtils.getPrecisionForType(b);
+ int scale1 = HiveDecimalUtils.getScaleForType(a);
+ int scale2 = HiveDecimalUtils.getScaleForType(b);
+ int intPart = Math.max(prec1 - scale1, prec2 - scale2);
+ int decPart = Math.max(scale1, scale2);
+ int prec = Math.min(intPart + decPart, HiveDecimal.MAX_PRECISION);
+ int scale = Math.min(decPart, HiveDecimal.MAX_PRECISION - intPart);
+ return TypeInfoFactory.getDecimalTypeInfo(prec, scale);
+ }
+
+ public static DecimalTypeInfo getDecimalTypeForPrimitiveCategory(PrimitiveTypeInfo a) {
+ if (a instanceof DecimalTypeInfo) return (DecimalTypeInfo)a;
+ int prec = HiveDecimalUtils.getPrecisionForType(a);
+ int scale = HiveDecimalUtils.getScaleForType(a);
+ prec = Math.min(prec, HiveDecimal.MAX_PRECISION);
+ scale = Math.min(scale, HiveDecimal.MAX_PRECISION - (prec - scale));
+ return TypeInfoFactory.getDecimalTypeInfo(prec, scale);
+ }
}
Modified: hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/VarcharTypeInfo.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/VarcharTypeInfo.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/VarcharTypeInfo.java (original)
+++ hive/branches/llap/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/VarcharTypeInfo.java Tue Oct 14 19:06:45 2014
@@ -25,6 +25,7 @@ public class VarcharTypeInfo extends Bas
// no-arg constructor to make kyro happy.
public VarcharTypeInfo() {
+ super(serdeConstants.VARCHAR_TYPE_NAME);
}
public VarcharTypeInfo(int length) {
Modified: hive/branches/llap/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java (original)
+++ hive/branches/llap/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java Tue Oct 14 19:06:45 2014
@@ -475,6 +475,67 @@ public class TestAvroDeserializer {
assertTrue(theMap2.containsKey("mu"));
assertEquals(null, theMap2.get("mu"));
}
+
+ @Test
+ public void canDeserializeMapsWithJavaLangStringKeys() throws IOException, SerDeException {
+ // Ensures maps can be deserialized when avro.java.string=String.
+ // See http://stackoverflow.com/a/19868919/312944 for why that might be used.
+ String schemaString = "{\n" +
+ " \"namespace\": \"testing\",\n" +
+ " \"name\": \"oneMap\",\n" +
+ " \"type\": \"record\",\n" +
+ " \"fields\": [\n" +
+ " {\n" +
+ " \"name\":\"aMap\",\n" +
+ " \"type\":{\"type\":\"map\",\n" +
+ " \"avro.java.string\":\"String\",\n" +
+ " \"values\":\"long\"}\n" +
+ "\t}\n" +
+ " ]\n" +
+ "}";
+ Schema s = AvroSerdeUtils.getSchemaFor(schemaString);
+ GenericData.Record record = new GenericData.Record(s);
+
+ Map<String, Long> m = new Hashtable<String, Long>();
+ m.put("one", 1l);
+ m.put("two", 2l);
+ m.put("three", 3l);
+
+ record.put("aMap", m);
+ assertTrue(GENERIC_DATA.validate(s, record));
+ System.out.println("record = " + record);
+
+ AvroGenericRecordWritable garw = Utils.serializeAndDeserializeRecord(record);
+
+ AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
+
+ AvroDeserializer de = new AvroDeserializer();
+
+ ArrayList<Object> row = (ArrayList<Object>)de.deserialize(aoig.getColumnNames(),
+ aoig.getColumnTypes(), garw, s);
+ assertEquals(1, row.size());
+ Object theMapObject = row.get(0);
+ assertTrue(theMapObject instanceof Map);
+ Map theMap = (Map)theMapObject;
+
+ // Verify the raw object that's been created
+ assertEquals(1l, theMap.get("one"));
+ assertEquals(2l, theMap.get("two"));
+ assertEquals(3l, theMap.get("three"));
+
+ // Verify that the provided object inspector can pull out these same values
+ StandardStructObjectInspector oi =
+ (StandardStructObjectInspector)aoig.getObjectInspector();
+
+ List<Object> z = oi.getStructFieldsDataAsList(row);
+ assertEquals(1, z.size());
+ StructField fieldRef = oi.getStructFieldRef("amap");
+
+ Map theMap2 = (Map)oi.getStructFieldData(row, fieldRef);
+ assertEquals(1l, theMap2.get("one"));
+ assertEquals(2l, theMap2.get("two"));
+ assertEquals(3l, theMap2.get("three"));
+ }
private void verifyNullableType(GenericData.Record record, Schema s, String fieldName,
String expected) throws SerDeException, IOException {
Modified: hive/branches/llap/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestTypeInfoToSchema.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestTypeInfoToSchema.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestTypeInfoToSchema.java (original)
+++ hive/branches/llap/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestTypeInfoToSchema.java Tue Oct 14 19:06:45 2014
@@ -19,9 +19,11 @@
package org.apache.hadoop.hive.serde2.avro;
import com.google.common.io.Resources;
+
import org.junit.Assert;
import org.apache.avro.Schema;
import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
@@ -44,16 +46,28 @@ public class TestTypeInfoToSchema {
private static Logger LOGGER = Logger.getLogger(TestTypeInfoToSchema.class);
private static final List<String> COLUMN_NAMES = Arrays.asList("testCol");
- private static final TypeInfo STRING = TypeInfoFactory.getPrimitiveTypeInfo("string");
- private static final TypeInfo INT = TypeInfoFactory.getPrimitiveTypeInfo("int");
- private static final TypeInfo BOOLEAN = TypeInfoFactory.getPrimitiveTypeInfo("boolean");
- private static final TypeInfo LONG = TypeInfoFactory.getPrimitiveTypeInfo("bigint");
- private static final TypeInfo FLOAT = TypeInfoFactory.getPrimitiveTypeInfo("float");
- private static final TypeInfo DOUBLE = TypeInfoFactory.getPrimitiveTypeInfo("double");
- private static final TypeInfo BINARY = TypeInfoFactory.getPrimitiveTypeInfo("binary");
- private static final TypeInfo BYTE = TypeInfoFactory.getPrimitiveTypeInfo("tinyint");
- private static final TypeInfo SHORT = TypeInfoFactory.getPrimitiveTypeInfo("smallint");
- private static final TypeInfo VOID = TypeInfoFactory.getPrimitiveTypeInfo("void");
+ private static final TypeInfo STRING = TypeInfoFactory.getPrimitiveTypeInfo(
+ serdeConstants.STRING_TYPE_NAME);
+ private static final TypeInfo INT = TypeInfoFactory.getPrimitiveTypeInfo(
+ serdeConstants.INT_TYPE_NAME);
+ private static final TypeInfo BOOLEAN = TypeInfoFactory.getPrimitiveTypeInfo(
+ serdeConstants.BOOLEAN_TYPE_NAME);
+ private static final TypeInfo LONG = TypeInfoFactory.getPrimitiveTypeInfo(
+ serdeConstants.BIGINT_TYPE_NAME);
+ private static final TypeInfo FLOAT = TypeInfoFactory.getPrimitiveTypeInfo(
+ serdeConstants.FLOAT_TYPE_NAME);
+ private static final TypeInfo DOUBLE = TypeInfoFactory.getPrimitiveTypeInfo(
+ serdeConstants.DOUBLE_TYPE_NAME);
+ private static final TypeInfo BINARY = TypeInfoFactory.getPrimitiveTypeInfo(
+ serdeConstants.BINARY_TYPE_NAME);
+ private static final TypeInfo BYTE = TypeInfoFactory.getPrimitiveTypeInfo(
+ serdeConstants.TINYINT_TYPE_NAME);
+ private static final TypeInfo SHORT = TypeInfoFactory.getPrimitiveTypeInfo(
+ serdeConstants.SMALLINT_TYPE_NAME);
+ private static final TypeInfo VOID = TypeInfoFactory.getPrimitiveTypeInfo(
+ serdeConstants.VOID_TYPE_NAME);
+ private static final TypeInfo DATE = TypeInfoFactory.getPrimitiveTypeInfo(
+ serdeConstants.DATE_TYPE_NAME);
private static final int PRECISION = 4;
private static final int SCALE = 2;
private static final TypeInfo DECIMAL = TypeInfoFactory.getPrimitiveTypeInfo(
@@ -205,6 +219,41 @@ public class TestTypeInfoToSchema {
}
@Test
+ public void createAvroCharSchema() {
+ final String specificSchema = "{" +
+ "\"type\":\"string\"," +
+ "\"logicalType\":\"char\"," +
+ "\"maxLength\":" + CHAR_LEN + "}";
+ String expectedSchema = genSchema(specificSchema);
+
+ Assert.assertEquals("Test for char's avro schema failed",
+ expectedSchema, getAvroSchemaString(CHAR));
+ }
+
+ @Test
+ public void createAvroVarcharSchema() {
+ final String specificSchema = "{" +
+ "\"type\":\"string\"," +
+ "\"logicalType\":\"varchar\"," +
+ "\"maxLength\":" + CHAR_LEN + "}";
+ String expectedSchema = genSchema(specificSchema);
+
+ Assert.assertEquals("Test for varchar's avro schema failed",
+ expectedSchema, getAvroSchemaString(VARCHAR));
+ }
+
+ @Test
+ public void createAvroDateSchema() {
+ final String specificSchema = "{" +
+ "\"type\":\"int\"," +
+ "\"logicalType\":\"date\"}";
+ String expectedSchema = genSchema(specificSchema);
+
+ Assert.assertEquals("Test for date in avro schema failed",
+ expectedSchema, getAvroSchemaString(DATE));
+ }
+
+ @Test
public void createAvroListSchema() {
ListTypeInfo listTypeInfo = new ListTypeInfo();
listTypeInfo.setListElementTypeInfo(STRING);
@@ -313,6 +362,7 @@ public class TestTypeInfoToSchema {
names.add("field11");
names.add("field12");
names.add("field13");
+ names.add("field14");
structTypeInfo.setAllStructFieldNames(names);
ArrayList<TypeInfo> typeInfos = new ArrayList<TypeInfo>();
typeInfos.add(STRING);
@@ -327,6 +377,7 @@ public class TestTypeInfoToSchema {
typeInfos.add(DOUBLE);
typeInfos.add(BOOLEAN);
typeInfos.add(DECIMAL);
+ typeInfos.add(DATE);
typeInfos.add(VOID);
structTypeInfo.setAllStructFieldTypeInfos(typeInfos);
LOGGER.info("structTypeInfo is " + structTypeInfo);
Modified: hive/branches/llap/serde/src/test/org/apache/hadoop/hive/serde2/io/TestHiveCharWritable.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/serde/src/test/org/apache/hadoop/hive/serde2/io/TestHiveCharWritable.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/serde/src/test/org/apache/hadoop/hive/serde2/io/TestHiveCharWritable.java (original)
+++ hive/branches/llap/serde/src/test/org/apache/hadoop/hive/serde2/io/TestHiveCharWritable.java Tue Oct 14 19:06:45 2014
@@ -18,10 +18,20 @@
package org.apache.hadoop.hive.serde2.io;
-import junit.framework.TestCase;
+import com.google.code.tempusfugit.concurrency.annotations.*;
+import com.google.code.tempusfugit.concurrency.*;
+import org.junit.*;
+
+import static org.junit.Assert.*;
import org.apache.hadoop.hive.common.type.HiveChar;
-public class TestHiveCharWritable extends TestCase {
+public class TestHiveCharWritable {
+ @Rule public ConcurrentRule concurrentRule = new ConcurrentRule();
+ @Rule public RepeatingRule repeatingRule = new RepeatingRule();
+
+ @Test
+ @Concurrent(count=4)
+ @Repeating(repetition=100)
public void testConstructor() throws Exception {
HiveCharWritable hcw1 = new HiveCharWritable(new HiveChar("abc", 5));
assertEquals("abc ", hcw1.toString());
@@ -30,6 +40,9 @@ public class TestHiveCharWritable extend
assertEquals("abc ", hcw2.toString());
}
+ @Test
+ @Concurrent(count=4)
+ @Repeating(repetition=100)
public void testSet() throws Exception {
HiveCharWritable hcw1 = new HiveCharWritable();
@@ -70,18 +83,27 @@ public class TestHiveCharWritable extend
assertEquals("ab", hcw1.getTextValue().toString());
}
+ @Test
+ @Concurrent(count=4)
+ @Repeating(repetition=100)
public void testGetHiveChar() throws Exception {
HiveCharWritable hcw = new HiveCharWritable();
hcw.set("abcd", 10);
assertEquals("abcd ", hcw.getHiveChar().toString());
}
+ @Test
+ @Concurrent(count=4)
+ @Repeating(repetition=100)
public void testGetCharacterLength() throws Exception {
HiveCharWritable hcw = new HiveCharWritable();
hcw.set("abcd", 10);
assertEquals(4, hcw.getCharacterLength());
}
+ @Test
+ @Concurrent(count=4)
+ @Repeating(repetition=100)
public void testEnforceMaxLength() {
HiveCharWritable hcw1 = new HiveCharWritable();
hcw1.set("abcdefghij", 10);
@@ -92,6 +114,9 @@ public class TestHiveCharWritable extend
assertEquals("abcde", hcw1.toString());
}
+ @Test
+ @Concurrent(count=4)
+ @Repeating(repetition=100)
public void testComparison() throws Exception {
HiveCharWritable hcw1 = new HiveCharWritable();
HiveCharWritable hcw2 = new HiveCharWritable();
Modified: hive/branches/llap/serde/src/test/org/apache/hadoop/hive/serde2/io/TestHiveDecimalWritable.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/serde/src/test/org/apache/hadoop/hive/serde2/io/TestHiveDecimalWritable.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/serde/src/test/org/apache/hadoop/hive/serde2/io/TestHiveDecimalWritable.java (original)
+++ hive/branches/llap/serde/src/test/org/apache/hadoop/hive/serde2/io/TestHiveDecimalWritable.java Tue Oct 14 19:06:45 2014
@@ -18,7 +18,10 @@
package org.apache.hadoop.hive.serde2.io;
-import junit.framework.Assert;
+import com.google.code.tempusfugit.concurrency.annotations.*;
+import com.google.code.tempusfugit.concurrency.*;
+import org.junit.*;
+import static org.junit.Assert.*;
import java.math.BigDecimal;
import java.math.BigInteger;
@@ -29,8 +32,6 @@ import java.util.ArrayList;
import org.apache.hadoop.hive.common.type.Decimal128;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hive.common.util.Decimal128FastBuffer;
-import org.junit.Before;
-import org.junit.Test;
/**
* Unit tests for tsting the fast allocation-free conversion
@@ -38,14 +39,15 @@ import org.junit.Test;
*/
public class TestHiveDecimalWritable {
- private Decimal128FastBuffer scratch;
+ @Rule public ConcurrentRule concurrentRule = new ConcurrentRule();
+ @Rule public RepeatingRule repeatingRule = new RepeatingRule();
@Before
public void setUp() throws Exception {
- scratch = new Decimal128FastBuffer();
}
private void doTestFastStreamForHiveDecimal(String valueString) {
+ Decimal128FastBuffer scratch = new Decimal128FastBuffer();
BigDecimal value = new BigDecimal(valueString);
Decimal128 dec = new Decimal128();
dec.update(value);
@@ -61,21 +63,23 @@ public class TestHiveDecimalWritable {
BigDecimal readValue = hd.bigDecimalValue();
- Assert.assertEquals(value, readValue);
+ assertEquals(value, readValue);
// Now test fastUpdate from the same serialized HiveDecimal
Decimal128 decRead = new Decimal128().fastUpdateFromInternalStorage(
witness.getInternalStorage(), (short) witness.getScale());
- Assert.assertEquals(dec, decRead);
+ assertEquals(dec, decRead);
// Test fastUpdate from it's own (not fully compacted) serialized output
Decimal128 decReadSelf = new Decimal128().fastUpdateFromInternalStorage(
hdw.getInternalStorage(), (short) hdw.getScale());
- Assert.assertEquals(dec, decReadSelf);
+ assertEquals(dec, decReadSelf);
}
@Test
+ @Concurrent(count=4)
+ @Repeating(repetition=100)
public void testFastStreamForHiveDecimal() {
doTestFastStreamForHiveDecimal("0");
@@ -217,7 +221,10 @@ public class TestHiveDecimalWritable {
}
@Test
+ @Concurrent(count=4)
+ @Repeating(repetition=100)
public void testHive6594() {
+ Decimal128FastBuffer scratch = new Decimal128FastBuffer();
String[] vs = new String[] {
"-4033.445769230769",
"6984454.211097692"};
@@ -236,7 +243,7 @@ public class TestHiveDecimalWritable {
BigDecimal readValue = hd.bigDecimalValue();
- Assert.assertEquals(d.toBigDecimal().stripTrailingZeros(),
+ assertEquals(d.toBigDecimal().stripTrailingZeros(),
readValue.stripTrailingZeros());
}
}
Modified: hive/branches/llap/serde/src/test/org/apache/hadoop/hive/serde2/io/TestHiveVarcharWritable.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/serde/src/test/org/apache/hadoop/hive/serde2/io/TestHiveVarcharWritable.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/serde/src/test/org/apache/hadoop/hive/serde2/io/TestHiveVarcharWritable.java (original)
+++ hive/branches/llap/serde/src/test/org/apache/hadoop/hive/serde2/io/TestHiveVarcharWritable.java Tue Oct 14 19:06:45 2014
@@ -17,11 +17,21 @@
*/
package org.apache.hadoop.hive.serde2.io;
-import junit.framework.TestCase;
+import com.google.code.tempusfugit.concurrency.annotations.*;
+import com.google.code.tempusfugit.concurrency.*;
+import org.junit.*;
+import static org.junit.Assert.*;
+
import org.apache.hadoop.hive.common.type.HiveVarchar;
import java.io.*;
-public class TestHiveVarcharWritable extends TestCase {
+public class TestHiveVarcharWritable {
+ @Rule public ConcurrentRule concurrentRule = new ConcurrentRule();
+ @Rule public RepeatingRule repeatingRule = new RepeatingRule();
+
+ @Test
+ @Concurrent(count=4)
+ @Repeating(repetition=100)
public void testStringLength() throws Exception {
HiveVarcharWritable vc1 = new HiveVarcharWritable(new HiveVarchar("0123456789", 10));
assertEquals(10, vc1.getCharacterLength());
@@ -54,6 +64,9 @@ public class TestHiveVarcharWritable ext
assertEquals(6, vc1.getCharacterLength());
}
+ @Test
+ @Concurrent(count=4)
+ @Repeating(repetition=100)
public void testEnforceLength() throws Exception {
HiveVarcharWritable vc1 = new HiveVarcharWritable(new HiveVarchar("0123456789", 10));
assertEquals(10, vc1.getCharacterLength());
@@ -66,8 +79,11 @@ public class TestHiveVarcharWritable ext
vc1.enforceMaxLength(8);
assertEquals(8, vc1.getCharacterLength());
-}
+ }
+ @Test
+ @Concurrent(count=4)
+ @Repeating(repetition=100)
public void testComparison() throws Exception {
HiveVarcharWritable hc1 = new HiveVarcharWritable(new HiveVarchar("abcd", 20));
HiveVarcharWritable hc2 = new HiveVarcharWritable(new HiveVarchar("abcd", 20));
@@ -101,6 +117,9 @@ public class TestHiveVarcharWritable ext
assertFalse(0 == hc2.compareTo(hc1));
}
+ @Test
+ @Concurrent(count=4)
+ @Repeating(repetition=100)
public void testStringValue() throws Exception {
HiveVarcharWritable vc1 = new HiveVarcharWritable(new HiveVarchar("abcde", 20));
assertEquals("abcde", vc1.toString());
Modified: hive/branches/llap/serde/src/test/org/apache/hadoop/hive/serde2/io/TestTimestampWritable.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/serde/src/test/org/apache/hadoop/hive/serde2/io/TestTimestampWritable.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/serde/src/test/org/apache/hadoop/hive/serde2/io/TestTimestampWritable.java (original)
+++ hive/branches/llap/serde/src/test/org/apache/hadoop/hive/serde2/io/TestTimestampWritable.java Tue Oct 14 19:06:45 2014
@@ -17,6 +17,9 @@
*/
package org.apache.hadoop.hive.serde2.io;
+import com.google.code.tempusfugit.concurrency.annotations.*;
+import com.google.code.tempusfugit.concurrency.*;
+
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
@@ -32,15 +35,25 @@ import java.util.List;
import java.util.Random;
import java.util.TimeZone;
-import junit.framework.TestCase;
+import org.junit.*;
+import static org.junit.Assert.*;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
-public class TestTimestampWritable extends TestCase {
+public class TestTimestampWritable {
+
+ @Rule public ConcurrentRule concurrentRule = new ConcurrentRule();
+ @Rule public RepeatingRule repeatingRule = new RepeatingRule();
- private static DateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ private static ThreadLocal<DateFormat> DATE_FORMAT =
+ new ThreadLocal<DateFormat>() {
+ @Override
+ protected synchronized DateFormat initialValue() {
+ return new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ }
+ };
private static final int HAS_DECIMAL_MASK = 0x80000000;
@@ -64,14 +77,14 @@ public class TestTimestampWritable exten
private static long parseToMillis(String s) {
try {
- return DATE_FORMAT.parse(s).getTime();
+ return DATE_FORMAT.get().parse(s).getTime();
} catch (ParseException ex) {
throw new RuntimeException(ex);
}
}
- @Override
- protected void setUp() {
+ @Before
+ public void setUp() {
TimeZone.setDefault(TimeZone.getTimeZone("UTC"));
}
@@ -252,6 +265,9 @@ public class TestTimestampWritable exten
return tsw;
}
+ @Test
+ @Concurrent(count=4)
+ @Repeating(repetition=100)
public void testReverseNanos() {
assertEquals(0, reverseNanos(0));
assertEquals(120000000, reverseNanos(21));
@@ -265,6 +281,8 @@ public class TestTimestampWritable exten
* Test serializing and deserializing timestamps that can be represented by a number of seconds
* from 0 to 2147483647 since the UNIX epoch.
*/
+ @Test
+ @Concurrent(count=4)
public void testTimestampsWithinPositiveIntRange() throws IOException {
Random rand = new Random(294722773L);
for (int i = 0; i < 10000; ++i) {
@@ -281,6 +299,8 @@ public class TestTimestampWritable exten
* Test timestamps that don't necessarily fit between 1970 and 2038. This depends on HIVE-4525
* being fixed.
*/
+ @Test
+ @Concurrent(count=4)
public void testTimestampsOutsidePositiveIntRange() throws IOException {
Random rand = new Random(789149717L);
for (int i = 0; i < 10000; ++i) {
@@ -289,6 +309,8 @@ public class TestTimestampWritable exten
}
}
+ @Test
+ @Concurrent(count=4)
public void testTimestampsInFullRange() throws IOException {
Random rand = new Random(2904974913L);
for (int i = 0; i < 10000; ++i) {
@@ -296,6 +318,8 @@ public class TestTimestampWritable exten
}
}
+ @Test
+ @Concurrent(count=4)
public void testToFromDouble() {
Random rand = new Random(294729777L);
for (int nanosPrecision = 0; nanosPrecision <= 4; ++nanosPrecision) {
@@ -326,6 +350,8 @@ public class TestTimestampWritable exten
return HiveDecimal.create(d);
}
+ @Test
+ @Concurrent(count=4)
public void testDecimalToTimestampRandomly() {
Random rand = new Random(294729777L);
for (int i = 0; i < 10000; ++i) {
@@ -336,6 +362,9 @@ public class TestTimestampWritable exten
}
}
+ @Test
+ @Concurrent(count=4)
+ @Repeating(repetition=100)
public void testDecimalToTimestampCornerCases() {
Timestamp ts = new Timestamp(parseToMillis("1969-03-04 05:44:33"));
assertEquals(0, ts.getTime() % 1000);
@@ -347,6 +376,9 @@ public class TestTimestampWritable exten
}
}
+ @Test
+ @Concurrent(count=4)
+ @Repeating(repetition=100)
public void testSerializationFormatDirectly() throws IOException {
assertEquals("1970-01-01 00:00:00", fromIntAndVInts(0).toString());
assertEquals("1970-01-01 00:00:01", fromIntAndVInts(1).toString());
@@ -374,6 +406,9 @@ public class TestTimestampWritable exten
-3210 - 1, seconds >> 31).toString());
}
+ @Test
+ @Concurrent(count=4)
+ @Repeating(repetition=100)
public void testMaxSize() {
// This many bytes are necessary to store the reversed nanoseconds.
assertEquals(5, WritableUtils.getVIntSize(999999999));
@@ -396,6 +431,9 @@ public class TestTimestampWritable exten
// Therefore, the maximum total size of a serialized timestamp is 4 + 5 + 4 = 13.
}
+ @Test
+ @Concurrent(count=4)
+ @Repeating(repetition=100)
public void testMillisToSeconds() {
assertEquals(0, TimestampWritable.millisToSeconds(0));
assertEquals(-1, TimestampWritable.millisToSeconds(-1));
@@ -427,6 +465,9 @@ public class TestTimestampWritable exten
return result < 0 ? -1 : (result > 0 ? 1 : 0);
}
+ @Test
+ @Concurrent(count=4)
+ @Repeating(repetition=100)
public void testBinarySortable() {
Random rand = new Random(5972977L);
List<TimestampWritable> tswList = new ArrayList<TimestampWritable>();
Modified: hive/branches/llap/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java (original)
+++ hive/branches/llap/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java Tue Oct 14 19:06:45 2014
@@ -194,6 +194,113 @@ public class TestLazyArrayMapStruct exte
}
}
+ /*
+ * test LazyMap with bad entries, e.g., empty key or empty entries
+ * where '[' and ']' don't exist, only for notation purpose,
+ * STX with value of 2 as entry separator, ETX with 3 as key/value separator
+ * */
+ public void testLazyMapWithBadEntries() throws Throwable {
+ try {
+ {
+ // Map of String to String
+ Text nullSequence = new Text("");
+ ObjectInspector oi = LazyFactory.createLazyObjectInspector(
+ TypeInfoUtils.getTypeInfosFromTypeString("map<string,string>").get(
+ 0), new byte[] {'\2', '\3'}, 0, nullSequence,
+ false, (byte) 0);
+ LazyMap b = (LazyMap) LazyFactory.createLazyObject(oi);
+
+ //read friendly string: ak[EXT]av[STX]bk[ETX]bv[STX]ck[ETX]cv[STX]dk[ETX]dv
+ byte[] data = new byte[] {
+ 'a', 'k', '\3', 'a', 'v',
+ '\02', 'b', 'k', '\3', 'b', 'v',
+ '\02', 'c', 'k', '\3', 'c', 'v',
+ '\02', 'd', 'k', '\3', 'd', 'v'};
+ TestLazyPrimitive.initLazyObject(b, data, 0, data.length);
+
+ assertEquals(new Text("av"), ((LazyString) b
+ .getMapValueElement(new Text("ak"))).getWritableObject());
+ assertNull(b.getMapValueElement(new Text("-1")));
+ assertEquals(new Text("bv"), ((LazyString) b
+ .getMapValueElement(new Text("bk"))).getWritableObject());
+ assertEquals(new Text("cv"), ((LazyString) b
+ .getMapValueElement(new Text("ck"))).getWritableObject());
+ assertNull(b.getMapValueElement(new Text("-")));
+ assertEquals(new Text("dv"), ((LazyString) b
+ .getMapValueElement(new Text("dk"))).getWritableObject());
+ assertEquals(4, b.getMapSize());
+ }
+
+ {
+ // Map of String to String, LazyMap allows empty-string style key, e.g., {"" : null}
+ // or {"", ""}, but not null style key, e.g., {null:""}
+ Text nullSequence = new Text("");
+ ObjectInspector oi = LazyFactory.createLazyObjectInspector(
+ TypeInfoUtils.getTypeInfosFromTypeString("map<string,string>").get(
+ 0), new byte[] {'\2', '\3'}, 0, nullSequence,
+ false, (byte) 0);
+ LazyMap b = (LazyMap) LazyFactory.createLazyObject(oi);
+
+ //read friendly string: [STX]ak[EXT]av[STX]bk[ETX]bv[STX]ck[ETX]cv[STX]dk[ETX]dv
+ byte[] data = new byte[] {
+ '\02', 'a', 'k', '\3', 'a', 'v',
+ '\02', 'b', 'k', '\3', 'b', 'v',
+ '\02', 'c', 'k', '\3', 'c', 'v',
+ '\02', 'd', 'k', '\3', 'd', 'v'};
+ TestLazyPrimitive.initLazyObject(b, data, 0, data.length);
+
+ assertNull(b.getMapValueElement(new Text(""))); //{"" : null}
+ assertEquals(new Text("av"), ((LazyString) b
+ .getMapValueElement(new Text("ak"))).getWritableObject());
+ assertNull(b.getMapValueElement(new Text("-1")));
+ assertEquals(new Text("bv"), ((LazyString) b
+ .getMapValueElement(new Text("bk"))).getWritableObject());
+ assertEquals(new Text("cv"), ((LazyString) b
+ .getMapValueElement(new Text("ck"))).getWritableObject());
+ assertNull(b.getMapValueElement(new Text("-")));
+ assertEquals(new Text("dv"), ((LazyString) b
+ .getMapValueElement(new Text("dk"))).getWritableObject());
+ assertEquals(4, b.getMapSize());
+ }
+
+ {
+ // Map of String to String, LazyMap allows empty-string style key, e.g., {"" : null}
+ // or {"", ""}, but not null style key, e.g., {null:""}
+ Text nullSequence = new Text("");
+ ObjectInspector oi = LazyFactory.createLazyObjectInspector(
+ TypeInfoUtils.getTypeInfosFromTypeString("map<string,string>").get(
+ 0), new byte[] {'\2', '\3'}, 0, nullSequence,
+ false, (byte) 0);
+ LazyMap b = (LazyMap) LazyFactory.createLazyObject(oi);
+
+ //read friendly string: [ETX][STX]ak[EXT]av[STX]bk[ETX]bv[STX]ck[ETX]cv[STX]dk[ETX]dv
+ byte[] data = new byte[] {
+ '\03',
+ '\02', 'a', 'k', '\3', 'a', 'v',
+ '\02', 'b', 'k', '\3', 'b', 'v',
+ '\02', 'c', 'k', '\3', 'c', 'v',
+ '\02', 'd', 'k', '\3', 'd', 'v'};
+ TestLazyPrimitive.initLazyObject(b, data, 0, data.length);
+
+ assertNull(b.getMapValueElement(new Text(""))); //{"" : null}
+ assertEquals(new Text("av"), ((LazyString) b
+ .getMapValueElement(new Text("ak"))).getWritableObject());
+ assertNull(b.getMapValueElement(new Text("-1")));
+ assertEquals(new Text("bv"), ((LazyString) b
+ .getMapValueElement(new Text("bk"))).getWritableObject());
+ assertEquals(new Text("cv"), ((LazyString) b
+ .getMapValueElement(new Text("ck"))).getWritableObject());
+ assertNull(b.getMapValueElement(new Text("-")));
+ assertEquals(new Text("dv"), ((LazyString) b
+ .getMapValueElement(new Text("dk"))).getWritableObject());
+ assertEquals(4, b.getMapSize());
+ }
+ } catch(Throwable e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
/**
* Test the LazyMap class.
*/
Modified: hive/branches/llap/serde/src/test/resources/avro-struct.avsc
URL: http://svn.apache.org/viewvc/hive/branches/llap/serde/src/test/resources/avro-struct.avsc?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/serde/src/test/resources/avro-struct.avsc (original)
+++ hive/branches/llap/serde/src/test/resources/avro-struct.avsc Tue Oct 14 19:06:45 2014
@@ -4,11 +4,11 @@
"namespace":"",
"doc":"struct<field1:string,field2:char(5),field3:varchar(5),field4:binary,field5:tinyint,
field6:smallint,field7:int,field8:bigint,field9:float,field10:double,field11:boolean,
-field12:decimal(4,2),field13:void>",
+field12:decimal(4,2),field13:date,field14:void>",
"fields":[
{"name":"field1","type":["null","string"],"doc":"string","default":null},
-{"name":"field2","type":["null","string"],"doc":"char(5)","default":null},
-{"name":"field3","type":["null","string"],"doc":"varchar(5)","default":null},
+{"name":"field2","type":["null",{"type":"string","logicalType":"char","maxLength":5}],"doc":"char(5)","default":null},
+{"name":"field3","type":["null",{"type":"string","logicalType":"varchar","maxLength":5}],"doc":"varchar(5)","default":null},
{"name":"field4","type":["null","bytes"],"doc":"binary","default":null},
{"name":"field5","type":["null","int"],"doc":"tinyint","default":null},
{"name":"field6","type":["null","int"],"doc":"smallint","default":null},
@@ -17,8 +17,8 @@ field12:decimal(4,2),field13:void>",
{"name":"field9","type":["null","float"],"doc":"float","default":null},
{"name":"field10","type":["null","double"],"doc":"double","default":null},
{"name":"field11","type":["null","boolean"],"doc":"boolean","default":null},
-{"name":"field12","type":["null",{"type":"bytes","logicalType":"decimal","precision":4,
-"scale":2}],"doc":"decimal(4,2)","default":null},
-{"name":"field13","type":"null","doc":"void","default":null}
+{"name":"field12","type":["null",{"type":"bytes","logicalType":"decimal","precision":4,"scale":2}],"doc":"decimal(4,2)","default":null},
+{"name":"field13","type":["null",{"type":"int","logicalType":"date"}],"doc":"date","default":null},
+{"name":"field14","type":"null","doc":"void","default":null}
]
-}
\ No newline at end of file
+}
Modified: hive/branches/llap/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java (original)
+++ hive/branches/llap/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java Tue Oct 14 19:06:45 2014
@@ -23,11 +23,13 @@ import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.util.HashMap;
import java.util.Map;
+
import javax.security.auth.login.LoginException;
import javax.security.sasl.Sasl;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.shims.HadoopShims.KerberosNameShim;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge;
import org.apache.hadoop.security.UserGroupInformation;
@@ -136,15 +138,17 @@ public class HiveAuthFactory {
return transportFactory;
}
+ /**
+ * Returns the thrift processor factory for HiveServer2 running in binary mode
+ * @param service
+ * @return
+ * @throws LoginException
+ */
public TProcessorFactory getAuthProcFactory(ThriftCLIService service) throws LoginException {
- if ("http".equalsIgnoreCase(transportMode)) {
- return HttpAuthUtils.getAuthProcFactory(service);
+ if (authTypeStr.equalsIgnoreCase(AuthTypes.KERBEROS.getAuthName())) {
+ return KerberosSaslHelper.getKerberosProcessorFactory(saslServer, service);
} else {
- if (authTypeStr.equalsIgnoreCase(AuthTypes.KERBEROS.getAuthName())) {
- return KerberosSaslHelper.getKerberosProcessorFactory(saslServer, service);
- } else {
- return PlainSaslHelper.getPlainProcessorFactory(service);
- }
+ return PlainSaslHelper.getPlainProcessorFactory(service);
}
}
@@ -283,11 +287,12 @@ public class HiveAuthFactory {
public static void verifyProxyAccess(String realUser, String proxyUser, String ipAddress,
HiveConf hiveConf) throws HiveSQLException {
-
try {
UserGroupInformation sessionUgi;
if (ShimLoader.getHadoopShims().isSecurityEnabled()) {
- sessionUgi = ShimLoader.getHadoopShims().createProxyUser(realUser);
+ KerberosNameShim kerbName = ShimLoader.getHadoopShims().getKerberosNameShim(realUser);
+ String shortPrincipalName = kerbName.getServiceName();
+ sessionUgi = ShimLoader.getHadoopShims().createProxyUser(shortPrincipalName);
} else {
sessionUgi = ShimLoader.getHadoopShims().createRemoteUser(realUser, null);
}