You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/07/09 11:07:43 UTC
svn commit: r1501145 [13/13] - in /hive/branches/vectorization: ./
beeline/src/java/org/apache/hive/beeline/
beeline/src/test/org/apache/hive/beeline/src/test/
cli/src/java/org/apache/hadoop/hive/cli/
common/src/java/org/apache/hadoop/hive/common/metri...
Modified: hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java (original)
+++ hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java Tue Jul 9 09:07:35 2013
@@ -35,12 +35,12 @@ import org.apache.hadoop.hive.serde2.io.
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveWritableObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
@@ -516,17 +516,29 @@ public final class ObjectInspectorUtils
ObjectInspector keyOI = mapOI.getMapKeyObjectInspector();
ObjectInspector valueOI = mapOI.getMapValueObjectInspector();
Map<?, ?> map = mapOI.getMap(o);
- for (Map.Entry entry : map.entrySet()) {
+ for (Map.Entry<?,?> entry : map.entrySet()) {
r += hashCode(entry.getKey(), keyOI) ^
hashCode(entry.getValue(), valueOI);
}
return r;
}
case STRUCT:
+ int r = 0;
+ StructObjectInspector structOI = (StructObjectInspector)objIns;
+ List<? extends StructField> fields = structOI.getAllStructFieldRefs();
+ for (StructField field : fields) {
+ r = 31 * r + hashCode(structOI.getStructFieldData(o, field),
+ field.getFieldObjectInspector());
+ }
+ return r;
+
case UNION:
+ UnionObjectInspector uOI = (UnionObjectInspector)objIns;
+ byte tag = uOI.getTag(o);
+ return hashCode(uOI.getField(o), uOI.getObjectInspectors().get(tag));
+
default:
- throw new RuntimeException(
- "Hash code on complex types not supported yet.");
+ throw new RuntimeException("Unknown type: "+ objIns.getTypeName());
}
}
Modified: hive/branches/vectorization/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java (original)
+++ hive/branches/vectorization/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java Tue Jul 9 09:07:35 2013
@@ -19,13 +19,21 @@ package org.apache.hadoop.hive.serde2.la
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Iterator;
import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
import junit.framework.TestCase;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.SerDeUtils;
import org.apache.hadoop.hive.serde2.io.ByteWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.hadoop.io.IntWritable;
@@ -37,6 +45,10 @@ import org.apache.hadoop.io.Text;
*/
public class TestLazyArrayMapStruct extends TestCase {
+ // nesting level limits
+ static final int EXTENDED_LEVEL_THRESHOLD = 24;
+ static final int DEFAULT_LEVEL_THRESHOLD = 8;
+
/**
* Test the LazyArray class.
*/
@@ -325,4 +337,287 @@ public class TestLazyArrayMapStruct exte
throw e;
}
}
+
+ /**
+ * Test the LazyArray class with multiple levels of nesting
+ */
+ public void testLazyArrayNested() throws Throwable {
+ for(int i = 2; i < EXTENDED_LEVEL_THRESHOLD; i++ ){
+ testNestedinArrayAtLevelExtended(i, ObjectInspector.Category.LIST);
+ }
+ }
+
+ /**
+ * Test the LazyArray class with multiple levels of nesting
+ */
+ public void testLazyArrayNestedExceedLimit() throws Throwable {
+ checkExtendedLimitExceeded(EXTENDED_LEVEL_THRESHOLD, ObjectInspector.Category.LIST);
+ }
+
+ private void checkExtendedLimitExceeded(int maxLevel, Category type) {
+ boolean foundException = false;
+ try {
+ testNestedinArrayAtLevelExtended(maxLevel, type);
+ }catch (SerDeException serdeEx){
+ foundException = true;
+ }
+ assertTrue("Got exception for exceeding nesting limit" , foundException);
+ }
+
+ /**
+ * Test the LazyArray class with multiple levels of nesting, when nesting
+ * levels are not extended
+ */
+ public void testLazyArrayNestedExceedLimitNotExtended() throws Throwable {
+ checkNotExtendedLimitExceeded(DEFAULT_LEVEL_THRESHOLD,
+ ObjectInspector.Category.LIST);
+ }
+
+ /**
+ * Test the LazyMap class with multiple levels of nesting, when nesting
+ * levels are not extended
+ */
+ public void testLazyMapNestedExceedLimitNotExtended() throws Throwable {
+ checkNotExtendedLimitExceeded(DEFAULT_LEVEL_THRESHOLD-1,
+ ObjectInspector.Category.MAP);
+ }
+
+ /**
+ * Test the LazyMap class with multiple levels of nesting, when nesting
+ * levels are not extended
+ */
+ public void testLazyStructNestedExceedLimitNotExtended() throws Throwable {
+ checkNotExtendedLimitExceeded(DEFAULT_LEVEL_THRESHOLD,
+ ObjectInspector.Category.STRUCT);
+ }
+
+ /**
+ * Test the LazyMap class with multiple levels of nesting, when nesting
+ * levels are not extended
+ */
+ public void testLazyUnionNestedExceedLimitNotExtended() throws Throwable {
+ checkNotExtendedLimitExceeded(DEFAULT_LEVEL_THRESHOLD,
+ ObjectInspector.Category.UNION);
+ }
+
+ private void checkNotExtendedLimitExceeded(int maxLevel, Category type) {
+ boolean foundException = false;
+ try {
+ testNestedinArrayAtLevel(maxLevel, type, new Properties());
+ }catch (SerDeException serdeEx){
+ foundException = true;
+ }
+ assertTrue("Expected exception for exceeding nesting limit" , foundException);
+ }
+
+ /**
+ * Test the LazyMap class with multiple levels of nesting
+ */
+ public void testLazyMapNested() throws Throwable {
+ //map max nesting level is one less because it uses an additional separator
+ for(int i = 2; i < EXTENDED_LEVEL_THRESHOLD - 1; i++ ){
+ testNestedinArrayAtLevelExtended(i, ObjectInspector.Category.MAP);
+ }
+ }
+
+ /**
+ * Test the LazyMap class with multiple levels of nesting
+ */
+ public void testLazyMapNestedExceedLimit() throws Throwable {
+ //map max nesting level is one less because it uses an additional separator
+ checkExtendedLimitExceeded(EXTENDED_LEVEL_THRESHOLD - 1, ObjectInspector.Category.MAP);
+ }
+
+ /**
+ * Test the LazyUnion class with multiple levels of nesting
+ */
+ public void testLazyUnionNested() throws Throwable {
+ for(int i = 2; i < EXTENDED_LEVEL_THRESHOLD; i++ ){
+ testNestedinArrayAtLevelExtended(i, ObjectInspector.Category.UNION);
+ }
+ }
+
+ /**
+ * Test the LazyUnion class with multiple levels of nesting
+ */
+ public void testLazyUnionNestedExceedLimit() throws Throwable {
+ checkExtendedLimitExceeded(EXTENDED_LEVEL_THRESHOLD, ObjectInspector.Category.UNION);
+ }
+
+ /**
+ * Test the LazyStruct class with multiple levels of nesting
+ */
+ public void testLazyStructNested() throws Throwable {
+ for(int i = 2; i < EXTENDED_LEVEL_THRESHOLD; i++ ){
+ testNestedinArrayAtLevelExtended(i, ObjectInspector.Category.STRUCT);
+ }
+ }
+
+ /**
+ * Verify the serialized format for given type dtype, when it is nested in an
+ * array with nestingLevel levels. with extended nesting enabled.
+ * @param nestingLevel
+ * @param dtype
+ * @throws SerDeException
+ */
+ private void testNestedinArrayAtLevelExtended(int nestingLevel,
+ ObjectInspector.Category dtype) throws SerDeException {
+ Properties tableProp = new Properties();
+ tableProp.setProperty(LazySimpleSerDe.SERIALIZATION_EXTEND_NESTING_LEVELS, "true");
+ testNestedinArrayAtLevel(nestingLevel, dtype, tableProp);
+ }
+
+ /**
+ * Test the LazyStruct class with multiple levels of nesting
+ */
+ public void testLazyStructNestedExceedLimit() throws Throwable {
+ checkExtendedLimitExceeded(EXTENDED_LEVEL_THRESHOLD, ObjectInspector.Category.STRUCT);
+ }
+
+ /**
+ * @param nestingLevel
+ * @param dtype
+ * @param tableProp
+ * @throws SerDeException
+ */
+ private void testNestedinArrayAtLevel(int nestingLevel,
+ ObjectInspector.Category dtype, Properties tableProp) throws SerDeException {
+
+ //create type with nestingLevel levels of nesting
+ //set inner schema for dtype
+ String inSchema = null;
+ switch(dtype){
+ case LIST:
+ inSchema = "array<tinyint>";
+ break;
+ case MAP:
+ inSchema = "map<string,int>";
+ break;
+ case STRUCT:
+ inSchema = "struct<s:string,i:tinyint>";
+ break;
+ case UNION:
+ inSchema = "uniontype<string,tinyint>";
+ break;
+ default :
+ fail("type not supported by test case");
+ }
+
+ StringBuilder schema = new StringBuilder(inSchema);
+ for(int i = 0; i < nestingLevel - 1; i++){
+ schema.insert(0, "array<");
+ schema.append(">");
+ }
+ System.err.println("Testing nesting level " + nestingLevel +
+ ". Using schema " + schema);
+
+
+ // Create the SerDe
+ LazySimpleSerDe serDe = new LazySimpleSerDe();
+ Configuration conf = new Configuration();
+ tableProp.setProperty("columns", "narray");
+ tableProp.setProperty("columns.types", schema.toString());
+ serDe.initialize(conf, tableProp);
+
+ //create the serialized string for type
+ byte[] separators = serDe.serdeParams.getSeparators();
+ System.err.println("Using separator " + (char)separators[nestingLevel]);
+ byte [] serializedRow = null;
+ switch(dtype){
+ case LIST:
+ serializedRow = new byte[] {'8',separators[nestingLevel],'9'};
+ break;
+ case MAP:
+ byte kvSep = separators[nestingLevel+1];
+ byte kvPairSep = separators[nestingLevel];
+ serializedRow = new byte[] {'1', kvSep, '1', kvPairSep, '2', kvSep, '2'};
+ break;
+ case STRUCT:
+ serializedRow = new byte[] {'8',separators[nestingLevel],'9'};
+ break;
+ case UNION:
+ serializedRow = new byte[] {'0',separators[nestingLevel],'9'};
+ break;
+ default :
+ fail("type not supported by test case");
+ }
+
+
+ //create LazyStruct with serialized string with expected separators
+ StructObjectInspector oi = (StructObjectInspector) serDe
+ .getObjectInspector();
+ LazyStruct struct = (LazyStruct) LazyFactory.createLazyObject(oi);
+
+ TestLazyPrimitive.initLazyObject(struct, serializedRow, 0, serializedRow.length);
+
+
+ //Get fields out of the lazy struct and check if they match expected
+ // results
+ //Get first level array
+ LazyArray array = (LazyArray) struct.getField(0);
+
+ //Peel off the n-1 levels to get to the underlying array
+ for(int i = 0; i < nestingLevel - 2; i++){
+ array = (LazyArray) array.getListElementObject(0);
+ }
+
+ //verify the serialized format for dtype
+ switch(dtype){
+ case LIST:
+ LazyArray array1 = (LazyArray) array.getListElementObject(0);
+ //check elements of the innermost array
+ assertEquals(2, array1.getListLength());
+ assertEquals(new ByteWritable((byte) 8), ((LazyByte) array1
+ .getListElementObject(0)).getWritableObject());
+ assertEquals(new ByteWritable((byte) 9), ((LazyByte) array1
+ .getListElementObject(1)).getWritableObject());
+ break;
+
+ case MAP:
+ LazyMap lazyMap = (LazyMap) array.getListElementObject(0);
+ Map map = lazyMap.getMap();
+ System.err.println(map);
+ assertEquals(2, map.size());
+ Iterator<Map.Entry<LazyString, LazyInteger>> it = map.entrySet().iterator();
+
+ Entry<LazyString, LazyInteger> e1 = it.next();
+ assertEquals(e1.getKey().getWritableObject(), new Text(new byte[]{'1'}) );
+ assertEquals(e1.getValue().getWritableObject(), new IntWritable(1) );
+
+ Entry<LazyString, LazyInteger> e2 = it.next();
+ assertEquals(e2.getKey().getWritableObject(), new Text(new byte[]{'2'}) );
+ assertEquals(e2.getValue().getWritableObject(), new IntWritable(2) );
+ break;
+
+ case STRUCT:
+ LazyStruct innerStruct = (LazyStruct) array.getListElementObject(0);
+ //check elements of the innermost struct
+ assertEquals(2, innerStruct.getFieldsAsList().size());
+ assertEquals(new Text(new byte[]{'8'}),
+ ((LazyString) innerStruct.getField(0)).getWritableObject());
+ assertEquals(new ByteWritable((byte) 9),
+ ((LazyByte) innerStruct.getField(1)).getWritableObject());
+ break;
+
+ case UNION:
+ LazyUnion lazyUnion = (LazyUnion) array.getListElementObject(0);
+ //check elements of the innermost union
+ assertEquals(new Text(new byte[]{'9'}),
+ ((LazyString)lazyUnion.getField()).getWritableObject());
+ break;
+
+
+ default :
+ fail("type not supported by test case");
+ }
+
+ //test serialization
+ Text serializedText =
+ (Text) serDe.serialize(struct.getObject(), serDe.getObjectInspector());
+ org.junit.Assert.assertArrayEquals(serializedRow, serializedText.getBytes());
+
+ }
+
+
+
}
Modified: hive/branches/vectorization/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyPrimitive.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyPrimitive.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyPrimitive.java (original)
+++ hive/branches/vectorization/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyPrimitive.java Tue Jul 9 09:07:35 2013
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hive.serde2.lazy;
+import java.sql.Timestamp;
+
import junit.framework.TestCase;
import org.apache.hadoop.hive.serde2.ByteStream;
@@ -388,6 +390,24 @@ public class TestLazyPrimitive extends T
assertEquals(new BytesWritable(new byte[] {}), ba.getWritableObject());
}
+ public void testLazyTimestamp() throws Throwable {
+ LazyTimestamp t = new LazyTimestamp(LazyPrimitiveObjectInspectorFactory.LAZY_TIMESTAMP_OBJECT_INSPECTOR);
+ String nullDate = "NULL";
+ byte[] nullBytes = nullDate.getBytes();
+ initLazyObject(t, nullBytes, 0, nullBytes.length);
+ assertEquals(true, t.isNull);
+ String sampleDate = "2013-02-12 21:04:58";
+ byte[] good2013 = sampleDate.getBytes();
+ initLazyObject(t, good2013, 0, good2013.length);
+ assertEquals(false, t.isNull);
+ assertEquals(Timestamp.valueOf(sampleDate),
+ t.getWritableObject().getTimestamp());
+ String badDate = "2013-02-12 21:04:XX";
+ byte[] bad2013 = badDate.getBytes();
+ initLazyObject(t, bad2013, 0, bad2013.length);
+ assertEquals(true, t.isNull);
+ }
+
public void testLazyIntegerWrite() throws Throwable {
try {
ByteStream.Output out = new ByteStream.Output();
Modified: hive/branches/vectorization/service/if/TCLIService.thrift
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/if/TCLIService.thrift?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/service/if/TCLIService.thrift (original)
+++ hive/branches/vectorization/service/if/TCLIService.thrift Tue Jul 9 09:07:35 2013
@@ -57,21 +57,23 @@ enum TTypeId {
STRUCT_TYPE,
UNION_TYPE,
USER_DEFINED_TYPE,
- DECIMAL_TYPE
+ DECIMAL_TYPE,
+ NULL_TYPE,
}
const set<TTypeId> PRIMITIVE_TYPES = [
- TTypeId.BOOLEAN_TYPE
- TTypeId.TINYINT_TYPE
- TTypeId.SMALLINT_TYPE
- TTypeId.INT_TYPE
- TTypeId.BIGINT_TYPE
- TTypeId.FLOAT_TYPE
- TTypeId.DOUBLE_TYPE
- TTypeId.STRING_TYPE
- TTypeId.TIMESTAMP_TYPE
+ TTypeId.BOOLEAN_TYPE,
+ TTypeId.TINYINT_TYPE,
+ TTypeId.SMALLINT_TYPE,
+ TTypeId.INT_TYPE,
+ TTypeId.BIGINT_TYPE,
+ TTypeId.FLOAT_TYPE,
+ TTypeId.DOUBLE_TYPE,
+ TTypeId.STRING_TYPE,
+ TTypeId.TIMESTAMP_TYPE,
TTypeId.BINARY_TYPE,
- TTypeId.DECIMAL_TYPE
+ TTypeId.DECIMAL_TYPE,
+ TTypeId.NULL_TYPE
]
const set<TTypeId> COMPLEX_TYPES = [
@@ -101,8 +103,9 @@ const map<TTypeId,string> TYPE_NAMES = {
TTypeId.ARRAY_TYPE: "ARRAY",
TTypeId.MAP_TYPE: "MAP",
TTypeId.STRUCT_TYPE: "STRUCT",
- TTypeId.UNION_TYPE: "UNIONTYPE"
- TTypeId.DECIMAL_TYPE: "DECIMAL"
+ TTypeId.UNION_TYPE: "UNIONTYPE",
+ TTypeId.DECIMAL_TYPE: "DECIMAL",
+ TTypeId.NULL_TYPE: "NULL"
}
// Thrift does not support recursively defined types or forward declarations,
@@ -285,7 +288,7 @@ union TColumnValue {
4: TI32Value i32Val // INT
5: TI64Value i64Val // BIGINT, TIMESTAMP
6: TDoubleValue doubleVal // FLOAT, DOUBLE
- 7: TStringValue stringVal // STRING, LIST, MAP, STRUCT, UNIONTYPE, BINARY, DECIMAL
+ 7: TStringValue stringVal // STRING, LIST, MAP, STRUCT, UNIONTYPE, BINARY, DECIMAL, NULL
}
// Represents a row in a rowset.
Modified: hive/branches/vectorization/service/src/gen/thrift/gen-cpp/TCLIService_constants.cpp
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/gen/thrift/gen-cpp/TCLIService_constants.cpp?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/service/src/gen/thrift/gen-cpp/TCLIService_constants.cpp (original)
+++ hive/branches/vectorization/service/src/gen/thrift/gen-cpp/TCLIService_constants.cpp Tue Jul 9 09:07:35 2013
@@ -22,6 +22,7 @@ TCLIServiceConstants::TCLIServiceConstan
PRIMITIVE_TYPES.insert((TTypeId::type)8);
PRIMITIVE_TYPES.insert((TTypeId::type)9);
PRIMITIVE_TYPES.insert((TTypeId::type)15);
+ PRIMITIVE_TYPES.insert((TTypeId::type)16);
COMPLEX_TYPES.insert((TTypeId::type)10);
COMPLEX_TYPES.insert((TTypeId::type)11);
@@ -47,6 +48,7 @@ TCLIServiceConstants::TCLIServiceConstan
TYPE_NAMES.insert(std::make_pair((TTypeId::type)12, "STRUCT"));
TYPE_NAMES.insert(std::make_pair((TTypeId::type)13, "UNIONTYPE"));
TYPE_NAMES.insert(std::make_pair((TTypeId::type)15, "DECIMAL"));
+ TYPE_NAMES.insert(std::make_pair((TTypeId::type)16, "NULL"));
}
Modified: hive/branches/vectorization/service/src/gen/thrift/gen-cpp/TCLIService_types.cpp
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/gen/thrift/gen-cpp/TCLIService_types.cpp?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/service/src/gen/thrift/gen-cpp/TCLIService_types.cpp (original)
+++ hive/branches/vectorization/service/src/gen/thrift/gen-cpp/TCLIService_types.cpp Tue Jul 9 09:07:35 2013
@@ -34,7 +34,8 @@ int _kTTypeIdValues[] = {
TTypeId::STRUCT_TYPE,
TTypeId::UNION_TYPE,
TTypeId::USER_DEFINED_TYPE,
- TTypeId::DECIMAL_TYPE
+ TTypeId::DECIMAL_TYPE,
+ TTypeId::NULL_TYPE
};
const char* _kTTypeIdNames[] = {
"BOOLEAN_TYPE",
@@ -52,9 +53,10 @@ const char* _kTTypeIdNames[] = {
"STRUCT_TYPE",
"UNION_TYPE",
"USER_DEFINED_TYPE",
- "DECIMAL_TYPE"
+ "DECIMAL_TYPE",
+ "NULL_TYPE"
};
-const std::map<int, const char*> _TTypeId_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(16, _kTTypeIdValues, _kTTypeIdNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+const std::map<int, const char*> _TTypeId_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(17, _kTTypeIdValues, _kTTypeIdNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
int _kTStatusCodeValues[] = {
TStatusCode::SUCCESS_STATUS,
Modified: hive/branches/vectorization/service/src/gen/thrift/gen-cpp/TCLIService_types.h
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/gen/thrift/gen-cpp/TCLIService_types.h?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/service/src/gen/thrift/gen-cpp/TCLIService_types.h (original)
+++ hive/branches/vectorization/service/src/gen/thrift/gen-cpp/TCLIService_types.h Tue Jul 9 09:07:35 2013
@@ -41,7 +41,8 @@ struct TTypeId {
STRUCT_TYPE = 12,
UNION_TYPE = 13,
USER_DEFINED_TYPE = 14,
- DECIMAL_TYPE = 15
+ DECIMAL_TYPE = 15,
+ NULL_TYPE = 16
};
};
Modified: hive/branches/vectorization/service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TCLIServiceConstants.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TCLIServiceConstants.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TCLIServiceConstants.java (original)
+++ hive/branches/vectorization/service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TCLIServiceConstants.java Tue Jul 9 09:07:35 2013
@@ -46,6 +46,7 @@ public class TCLIServiceConstants {
PRIMITIVE_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.TIMESTAMP_TYPE);
PRIMITIVE_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.BINARY_TYPE);
PRIMITIVE_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.DECIMAL_TYPE);
+ PRIMITIVE_TYPES.add(org.apache.hive.service.cli.thrift.TTypeId.NULL_TYPE);
}
public static final Set<TTypeId> COMPLEX_TYPES = new HashSet<TTypeId>();
@@ -80,6 +81,7 @@ public class TCLIServiceConstants {
TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.STRUCT_TYPE, "STRUCT");
TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.UNION_TYPE, "UNIONTYPE");
TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.DECIMAL_TYPE, "DECIMAL");
+ TYPE_NAMES.put(org.apache.hive.service.cli.thrift.TTypeId.NULL_TYPE, "NULL");
}
}
Modified: hive/branches/vectorization/service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTypeId.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTypeId.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTypeId.java (original)
+++ hive/branches/vectorization/service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTypeId.java Tue Jul 9 09:07:35 2013
@@ -27,7 +27,8 @@ public enum TTypeId implements org.apach
STRUCT_TYPE(12),
UNION_TYPE(13),
USER_DEFINED_TYPE(14),
- DECIMAL_TYPE(15);
+ DECIMAL_TYPE(15),
+ NULL_TYPE(16);
private final int value;
@@ -80,6 +81,8 @@ public enum TTypeId implements org.apach
return USER_DEFINED_TYPE;
case 15:
return DECIMAL_TYPE;
+ case 16:
+ return NULL_TYPE;
default:
return null;
}
Modified: hive/branches/vectorization/service/src/gen/thrift/gen-py/TCLIService/constants.py
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/gen/thrift/gen-py/TCLIService/constants.py?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/service/src/gen/thrift/gen-py/TCLIService/constants.py (original)
+++ hive/branches/vectorization/service/src/gen/thrift/gen-py/TCLIService/constants.py Tue Jul 9 09:07:35 2013
@@ -21,6 +21,7 @@ PRIMITIVE_TYPES = set([
8,
9,
15,
+ 16,
])
COMPLEX_TYPES = set([
10,
@@ -49,4 +50,5 @@ TYPE_NAMES = {
12 : "STRUCT",
13 : "UNIONTYPE",
15 : "DECIMAL",
+ 16 : "NULL",
}
Modified: hive/branches/vectorization/service/src/gen/thrift/gen-py/TCLIService/ttypes.py
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/gen/thrift/gen-py/TCLIService/ttypes.py?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/service/src/gen/thrift/gen-py/TCLIService/ttypes.py (original)
+++ hive/branches/vectorization/service/src/gen/thrift/gen-py/TCLIService/ttypes.py Tue Jul 9 09:07:35 2013
@@ -44,6 +44,7 @@ class TTypeId:
UNION_TYPE = 13
USER_DEFINED_TYPE = 14
DECIMAL_TYPE = 15
+ NULL_TYPE = 16
_VALUES_TO_NAMES = {
0: "BOOLEAN_TYPE",
@@ -62,6 +63,7 @@ class TTypeId:
13: "UNION_TYPE",
14: "USER_DEFINED_TYPE",
15: "DECIMAL_TYPE",
+ 16: "NULL_TYPE",
}
_NAMES_TO_VALUES = {
@@ -81,6 +83,7 @@ class TTypeId:
"UNION_TYPE": 13,
"USER_DEFINED_TYPE": 14,
"DECIMAL_TYPE": 15,
+ "NULL_TYPE": 16,
}
class TStatusCode:
Modified: hive/branches/vectorization/service/src/gen/thrift/gen-rb/t_c_l_i_service_constants.rb
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/gen/thrift/gen-rb/t_c_l_i_service_constants.rb?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/service/src/gen/thrift/gen-rb/t_c_l_i_service_constants.rb (original)
+++ hive/branches/vectorization/service/src/gen/thrift/gen-rb/t_c_l_i_service_constants.rb Tue Jul 9 09:07:35 2013
@@ -19,6 +19,7 @@ PRIMITIVE_TYPES = Set.new([
8,
9,
15,
+ 16,
])
COMPLEX_TYPES = Set.new([
@@ -50,5 +51,6 @@ TYPE_NAMES = {
12 => %q"STRUCT",
13 => %q"UNIONTYPE",
15 => %q"DECIMAL",
+ 16 => %q"NULL",
}
Modified: hive/branches/vectorization/service/src/gen/thrift/gen-rb/t_c_l_i_service_types.rb
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/gen/thrift/gen-rb/t_c_l_i_service_types.rb?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/service/src/gen/thrift/gen-rb/t_c_l_i_service_types.rb (original)
+++ hive/branches/vectorization/service/src/gen/thrift/gen-rb/t_c_l_i_service_types.rb Tue Jul 9 09:07:35 2013
@@ -29,8 +29,9 @@ module TTypeId
UNION_TYPE = 13
USER_DEFINED_TYPE = 14
DECIMAL_TYPE = 15
- VALUE_MAP = {0 => "BOOLEAN_TYPE", 1 => "TINYINT_TYPE", 2 => "SMALLINT_TYPE", 3 => "INT_TYPE", 4 => "BIGINT_TYPE", 5 => "FLOAT_TYPE", 6 => "DOUBLE_TYPE", 7 => "STRING_TYPE", 8 => "TIMESTAMP_TYPE", 9 => "BINARY_TYPE", 10 => "ARRAY_TYPE", 11 => "MAP_TYPE", 12 => "STRUCT_TYPE", 13 => "UNION_TYPE", 14 => "USER_DEFINED_TYPE", 15 => "DECIMAL_TYPE"}
- VALID_VALUES = Set.new([BOOLEAN_TYPE, TINYINT_TYPE, SMALLINT_TYPE, INT_TYPE, BIGINT_TYPE, FLOAT_TYPE, DOUBLE_TYPE, STRING_TYPE, TIMESTAMP_TYPE, BINARY_TYPE, ARRAY_TYPE, MAP_TYPE, STRUCT_TYPE, UNION_TYPE, USER_DEFINED_TYPE, DECIMAL_TYPE]).freeze
+ NULL_TYPE = 16
+ VALUE_MAP = {0 => "BOOLEAN_TYPE", 1 => "TINYINT_TYPE", 2 => "SMALLINT_TYPE", 3 => "INT_TYPE", 4 => "BIGINT_TYPE", 5 => "FLOAT_TYPE", 6 => "DOUBLE_TYPE", 7 => "STRING_TYPE", 8 => "TIMESTAMP_TYPE", 9 => "BINARY_TYPE", 10 => "ARRAY_TYPE", 11 => "MAP_TYPE", 12 => "STRUCT_TYPE", 13 => "UNION_TYPE", 14 => "USER_DEFINED_TYPE", 15 => "DECIMAL_TYPE", 16 => "NULL_TYPE"}
+ VALID_VALUES = Set.new([BOOLEAN_TYPE, TINYINT_TYPE, SMALLINT_TYPE, INT_TYPE, BIGINT_TYPE, FLOAT_TYPE, DOUBLE_TYPE, STRING_TYPE, TIMESTAMP_TYPE, BINARY_TYPE, ARRAY_TYPE, MAP_TYPE, STRUCT_TYPE, UNION_TYPE, USER_DEFINED_TYPE, DECIMAL_TYPE, NULL_TYPE]).freeze
end
module TStatusCode
Modified: hive/branches/vectorization/service/src/java/org/apache/hive/service/auth/LdapAuthenticationProviderImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/java/org/apache/hive/service/auth/LdapAuthenticationProviderImpl.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/service/src/java/org/apache/hive/service/auth/LdapAuthenticationProviderImpl.java (original)
+++ hive/branches/vectorization/service/src/java/org/apache/hive/service/auth/LdapAuthenticationProviderImpl.java Tue Jul 9 09:07:35 2013
@@ -29,13 +29,15 @@ import org.apache.hadoop.hive.conf.HiveC
public class LdapAuthenticationProviderImpl implements PasswdAuthenticationProvider {
- String ldapURL;
- String baseDN;
+ private final String ldapURL;
+ private final String baseDN;
+ private final String ldapDomain;
LdapAuthenticationProviderImpl () {
HiveConf conf = new HiveConf();
this.ldapURL = conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_URL);
this.baseDN = conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_BASEDN);
+ this.ldapDomain = conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_PLAIN_LDAP_DOMAIN);
}
@Override
@@ -46,6 +48,12 @@ public class LdapAuthenticationProviderI
env.put(Context.INITIAL_CONTEXT_FACTORY, "com.sun.jndi.ldap.LdapCtxFactory");
env.put(Context.PROVIDER_URL, ldapURL);
+ // If the domain is supplied, then append it. LDAP providers like Active Directory
+ // use a fully qualified user name like foo@bar.com.
+ if (ldapDomain != null) {
+ user = user + "@" + ldapDomain;
+ }
+
// setup the security principal
String bindDN;
if (baseDN != null) {
Modified: hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/ColumnValue.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/ColumnValue.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/ColumnValue.java (original)
+++ hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/ColumnValue.java Tue Jul 9 09:07:35 2013
@@ -36,6 +36,12 @@ import org.apache.hive.service.cli.thrif
*/
public class ColumnValue {
+ public static final TColumnValue NULL = new TColumnValue();
+
+ static {
+ NULL.setStringVal(new TStringValue());
+ }
+
// TODO: replace this with a non-Thrift implementation
private final TColumnValue tColumnValue;
Modified: hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/Row.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/Row.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/Row.java (original)
+++ hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/Row.java Tue Jul 9 09:07:35 2013
@@ -71,7 +71,11 @@ public class Row {
public TRow toTRow() {
TRow tRow = new TRow();
for (ColumnValue columnValue : values) {
- tRow.addToColVals(columnValue.toTColumnValue());
+ if (columnValue != null) {
+ tRow.addToColVals(columnValue.toTColumnValue());
+ } else {
+ tRow.addToColVals(ColumnValue.NULL);
+ }
}
return tRow;
}
Modified: hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/Type.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/Type.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/Type.java (original)
+++ hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/Type.java Tue Jul 9 09:07:35 2013
@@ -27,6 +27,9 @@ import org.apache.hive.service.cli.thrif
*
*/
public enum Type {
+ NULL_TYPE("VOID",
+ java.sql.Types.NULL,
+ TTypeId.NULL_TYPE),
BOOLEAN_TYPE("BOOLEAN",
java.sql.Types.BOOLEAN,
TTypeId.BOOLEAN_TYPE),
@@ -77,7 +80,7 @@ public enum Type {
java.sql.Types.VARCHAR,
TTypeId.STRING_TYPE,
true, false),
- USER_DEFINED_TYPE(null,
+ USER_DEFINED_TYPE("USER_DEFINED",
java.sql.Types.VARCHAR,
TTypeId.STRING_TYPE,
true, false);
@@ -123,6 +126,9 @@ public enum Type {
}
public static Type getType(String name) {
+ if (name == null) {
+ throw new IllegalArgumentException("Invalid type name: null");
+ }
for (Type type : values()) {
if (name.equalsIgnoreCase(type.name)) {
return type;
Modified: hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/GetSchemasOperation.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/GetSchemasOperation.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/GetSchemasOperation.java (original)
+++ hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/GetSchemasOperation.java Tue Jul 9 09:07:35 2013
@@ -36,7 +36,7 @@ public class GetSchemasOperation extends
private final String schemaName;
private static final TableSchema RESULT_SET_SCHEMA = new TableSchema()
- .addStringColumn("TABLE_SCHEMA", "Schema name.")
+ .addStringColumn("TABLE_SCHEM", "Schema name.")
.addStringColumn("TABLE_CATALOG", "Catalog name.");
private RowSet rowSet;
Modified: hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/GetTablesOperation.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/GetTablesOperation.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/GetTablesOperation.java (original)
+++ hive/branches/vectorization/service/src/java/org/apache/hive/service/cli/operation/GetTablesOperation.java Tue Jul 9 09:07:35 2013
@@ -46,7 +46,7 @@ public class GetTablesOperation extends
private static final TableSchema RESULT_SET_SCHEMA = new TableSchema()
.addStringColumn("TABLE_CAT", "Catalog name. NULL if not applicable.")
- .addStringColumn("TABLE_SCHEMA", "Schema name.")
+ .addStringColumn("TABLE_SCHEM", "Schema name.")
.addStringColumn("TABLE_NAME", "Table name.")
.addStringColumn("TABLE_TYPE", "The table type, e.g. \"TABLE\", \"VIEW\", etc.")
.addStringColumn("REMARKS", "Comments about the table.");