You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/09/20 02:33:01 UTC
svn commit: r1524874 [9/9] - in /hive/branches/vectorization: ./
cli/src/java/org/apache/hadoop/hive/cli/
cli/src/test/org/apache/hadoop/hive/cli/
common/src/java/org/apache/hadoop/hive/common/type/
common/src/java/org/apache/hadoop/hive/conf/ common/s...
Modified: hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantIntObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantIntObjectInspector.java?rev=1524874&r1=1524873&r2=1524874&view=diff
==============================================================================
--- hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantIntObjectInspector.java (original)
+++ hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantIntObjectInspector.java Fri Sep 20 00:32:55 2013
@@ -31,6 +31,9 @@ public class WritableConstantIntObjectIn
private IntWritable value;
+ protected WritableConstantIntObjectInspector() {
+ super();
+ }
WritableConstantIntObjectInspector(IntWritable value) {
super();
this.value = value;
Modified: hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantLongObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantLongObjectInspector.java?rev=1524874&r1=1524873&r2=1524874&view=diff
==============================================================================
--- hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantLongObjectInspector.java (original)
+++ hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantLongObjectInspector.java Fri Sep 20 00:32:55 2013
@@ -31,6 +31,9 @@ public class WritableConstantLongObjectI
private LongWritable value;
+ protected WritableConstantLongObjectInspector() {
+ super();
+ }
WritableConstantLongObjectInspector(LongWritable value) {
super();
this.value = value;
Modified: hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantShortObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantShortObjectInspector.java?rev=1524874&r1=1524873&r2=1524874&view=diff
==============================================================================
--- hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantShortObjectInspector.java (original)
+++ hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantShortObjectInspector.java Fri Sep 20 00:32:55 2013
@@ -31,6 +31,9 @@ public class WritableConstantShortObject
private ShortWritable value;
+ protected WritableConstantShortObjectInspector() {
+ super();
+ }
WritableConstantShortObjectInspector(ShortWritable value) {
super();
this.value = value;
Modified: hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantStringObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantStringObjectInspector.java?rev=1524874&r1=1524873&r2=1524874&view=diff
==============================================================================
--- hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantStringObjectInspector.java (original)
+++ hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantStringObjectInspector.java Fri Sep 20 00:32:55 2013
@@ -31,6 +31,9 @@ public class WritableConstantStringObjec
private Text value;
+ protected WritableConstantStringObjectInspector() {
+ super();
+ }
WritableConstantStringObjectInspector(Text value) {
super();
this.value = value;
Modified: hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantTimestampObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantTimestampObjectInspector.java?rev=1524874&r1=1524873&r2=1524874&view=diff
==============================================================================
--- hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantTimestampObjectInspector.java (original)
+++ hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantTimestampObjectInspector.java Fri Sep 20 00:32:55 2013
@@ -31,6 +31,9 @@ public class WritableConstantTimestampOb
private TimestampWritable value;
+ protected WritableConstantTimestampObjectInspector() {
+ super();
+ }
WritableConstantTimestampObjectInspector(TimestampWritable value) {
super();
this.value = value;
Modified: hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java?rev=1524874&r1=1524873&r2=1524874&view=diff
==============================================================================
--- hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java (original)
+++ hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java Fri Sep 20 00:32:55 2013
@@ -23,19 +23,21 @@ import java.io.Serializable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry;
/**
* There are limited number of Primitive Types. All Primitive Types are defined
* by TypeInfoFactory.isPrimitiveClass().
- *
+ *
* Always use the TypeInfoFactory to create new TypeInfo objects, instead of
* directly creating an instance of this class.
*/
-public final class PrimitiveTypeInfo extends TypeInfo implements Serializable {
+public class PrimitiveTypeInfo extends TypeInfo implements Serializable, PrimitiveTypeSpec {
private static final long serialVersionUID = 1L;
- private String typeName;
+ protected String typeName;
+ protected BaseTypeParams typeParams;
/**
* For java serialization use only.
@@ -59,7 +61,7 @@ public final class PrimitiveTypeInfo ext
}
public PrimitiveCategory getPrimitiveCategory() {
- return PrimitiveObjectInspectorUtils.getTypeEntryFromTypeName(typeName).primitiveCategory;
+ return getPrimitiveTypeEntry().primitiveCategory;
}
public Class<?> getPrimitiveWritableClass() {
@@ -81,6 +83,36 @@ public final class PrimitiveTypeInfo ext
}
/**
+ * If the type has type parameters (such as varchar length, or decimal precision/scale),
+ * then return the parameters for the type.
+ * @return A BaseTypeParams object representing the parameters for the type, or null
+ */
+ public BaseTypeParams getTypeParams() {
+ return typeParams;
+ }
+
+ /**
+ * Set the type parameters for the type.
+ * @param typeParams type parameters for the type
+ */
+ public void setTypeParams(BaseTypeParams typeParams) {
+ // Ideally could check here to make sure the type really supports parameters,
+ // however during deserialization some of the required fields are not set at the
+ // time that the type params are set. We would have to customize the way this class
+ // is serialized/deserialized for the check to work.
+ //if (typeParams != null && !getPrimitiveTypeEntry().isParameterized()) {
+ // throw new UnsupportedOperationException(
+ // "Attempting to add type parameters " + typeParams + " to type " + getTypeName());
+ //}
+ this.typeParams = typeParams;
+ }
+
+ public PrimitiveTypeEntry getPrimitiveTypeEntry() {
+ return PrimitiveObjectInspectorUtils.getTypeEntryFromTypeName(
+ TypeInfoUtils.getBaseName(typeName));
+ }
+
+ /**
* Compare if 2 TypeInfos are the same. We use TypeInfoFactory to cache
* TypeInfos, so we only need to compare the Object pointer.
*/
@@ -97,4 +129,8 @@ public final class PrimitiveTypeInfo ext
return typeName.hashCode();
}
+ @Override
+ public String toString() {
+ return typeName;
+ }
}
Modified: hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java?rev=1524874&r1=1524873&r2=1524874&view=diff
==============================================================================
--- hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java (original)
+++ hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java Fri Sep 20 00:32:55 2013
@@ -49,6 +49,15 @@ public abstract class TypeInfo implement
*/
public abstract String getTypeName();
+ /**
+ * String representing the qualified type name.
+ * Qualified types should override this method.
+ * @return
+ */
+ public String getQualifiedName() {
+ return getTypeName();
+ }
+
@Override
public String toString() {
return getTypeName();
Modified: hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java?rev=1524874&r1=1524873&r2=1524874&view=diff
==============================================================================
--- hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java (original)
+++ hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java Fri Sep 20 00:32:55 2013
@@ -22,8 +22,12 @@ import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry;
/**
* TypeInfoFactory can be used to create the TypeInfo object for any types.
@@ -33,7 +37,7 @@ import org.apache.hadoop.hive.serde2.obj
* objects that represents the same type.
*/
public final class TypeInfoFactory {
-
+ private static Log LOG = LogFactory.getLog(TypeInfoFactory.class);
static ConcurrentHashMap<String, TypeInfo> cachedPrimitiveTypeInfo = new ConcurrentHashMap<String, TypeInfo>();
private TypeInfoFactory() {
@@ -41,13 +45,42 @@ public final class TypeInfoFactory {
}
public static TypeInfo getPrimitiveTypeInfo(String typeName) {
- if (null == PrimitiveObjectInspectorUtils
- .getTypeEntryFromTypeName(typeName)) {
+ PrimitiveTypeEntry typeEntry = PrimitiveObjectInspectorUtils
+ .getTypeEntryFromTypeName(TypeInfoUtils.getBaseName(typeName));
+ if (null == typeEntry) {
throw new RuntimeException("Cannot getPrimitiveTypeInfo for " + typeName);
}
TypeInfo result = cachedPrimitiveTypeInfo.get(typeName);
if (result == null) {
- result = new PrimitiveTypeInfo(typeName);
+ TypeInfoUtils.PrimitiveParts parts = TypeInfoUtils.parsePrimitiveParts(typeName);
+ // Create params if there are any
+ if (parts.typeParams != null && parts.typeParams.length > 0) {
+ // The type string came with parameters. Parse and add to TypeInfo
+ try {
+ BaseTypeParams typeParams = PrimitiveTypeEntry.createTypeParams(
+ parts.typeName, parts.typeParams);
+ result = new PrimitiveTypeInfo(typeName);
+ ((PrimitiveTypeInfo) result).setTypeParams(typeParams);
+ } catch (Exception err) {
+ LOG.error(err);
+ throw new RuntimeException("Error creating type parameters for " + typeName
+ + ": " + err, err);
+ }
+ } else {
+ // No type params
+
+ // Prevent creation of varchar TypeInfo with no length specification.
+ // This can happen if an old-style UDF uses a varchar type either as an
+ // argument or return type in an evaluate() function, or other instances
+ // of using reflection-based methods for retrieving a TypeInfo.
+ if (typeEntry.primitiveCategory == PrimitiveCategory.VARCHAR) {
+ LOG.error("varchar type used with no type params");
+ throw new RuntimeException("varchar type used with no type params");
+ }
+
+ result = new PrimitiveTypeInfo(parts.typeName);
+ }
+
cachedPrimitiveTypeInfo.put(typeName, result);
}
return result;
@@ -66,6 +99,8 @@ public final class TypeInfoFactory {
public static final TypeInfo timestampTypeInfo = getPrimitiveTypeInfo(serdeConstants.TIMESTAMP_TYPE_NAME);
public static final TypeInfo binaryTypeInfo = getPrimitiveTypeInfo(serdeConstants.BINARY_TYPE_NAME);
public static final TypeInfo decimalTypeInfo = getPrimitiveTypeInfo(serdeConstants.DECIMAL_TYPE_NAME);
+ // Disallow usage of varchar without length specifier.
+ //public static final TypeInfo varcharTypeInfo = getPrimitiveTypeInfo(serdeConstants.VARCHAR_TYPE_NAME);
public static final TypeInfo unknownTypeInfo = getPrimitiveTypeInfo("unknown");
Modified: hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java?rev=1524874&r1=1524873&r2=1524874&view=diff
==============================================================================
--- hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java (original)
+++ hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java Fri Sep 20 00:32:55 2013
@@ -22,23 +22,27 @@ import java.lang.reflect.GenericArrayTyp
import java.lang.reflect.Method;
import java.lang.reflect.ParameterizedType;
import java.lang.reflect.Type;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry;
@@ -56,7 +60,7 @@ public final class TypeInfoUtils {
/**
* Return the extended TypeInfo from a Java type. By extended TypeInfo, we
* allow unknownType for java.lang.Object.
- *
+ *
* @param t
* The Java type.
* @param m
@@ -148,7 +152,7 @@ public final class TypeInfoUtils {
/**
* Get the parameter TypeInfo for a method.
- *
+ *
* @param size
* In case the last parameter of Method is an array, we will try to
* return a List<TypeInfo> with the specified size by repeating the
@@ -194,12 +198,46 @@ public final class TypeInfoUtils {
return typeInfos;
}
+ public static boolean hasParameters(String typeName) {
+ int idx = typeName.indexOf('(');
+ if (idx == -1) {
+ return false;
+ } else {
+ return true;
+ }
+ }
+
+ public static String getBaseName(String typeName) {
+ int idx = typeName.indexOf('(');
+ if (idx == -1) {
+ return typeName;
+ } else {
+ return typeName.substring(0, idx);
+ }
+ }
+
+ /**
+ * returns true if both TypeInfos are of primitive type, and the primitive category matches.
+ * @param ti1
+ * @param ti2
+ * @return
+ */
+ public static boolean doPrimitiveCategoriesMatch(TypeInfo ti1, TypeInfo ti2) {
+ if (ti1.getCategory() == Category.PRIMITIVE && ti2.getCategory() == Category.PRIMITIVE) {
+ if (((PrimitiveTypeInfo)ti1).getPrimitiveCategory()
+ == ((PrimitiveTypeInfo)ti2).getPrimitiveCategory()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
/**
* Parse a recursive TypeInfo list String. For example, the following inputs
* are valid inputs:
* "int,string,map<string,int>,list<map<int,list<string>>>,list<struct<a:int,b:string>>"
* The separators between TypeInfos can be ",", ":", or ";".
- *
+ *
* In order to use this class: TypeInfoParser parser = new
* TypeInfoParser("int,string"); ArrayList<TypeInfo> typeInfos =
* parser.parseTypeInfos();
@@ -225,7 +263,7 @@ public final class TypeInfoUtils {
* Tokenize the typeInfoString. The rule is simple: all consecutive
* alphadigits and '_', '.' are in one token, and all other characters are
* one character per token.
- *
+ *
* tokenize("map<int,string>") should return
* ["map","<","int",",","string",">"]
*/
@@ -281,6 +319,14 @@ public final class TypeInfoUtils {
return typeInfos;
}
+ private Token peek() {
+ if (iToken < typeInfoTokens.size()) {
+ return typeInfoTokens.get(iToken);
+ } else {
+ return null;
+ }
+ }
+
private Token expect(String item) {
return expect(item, null);
}
@@ -320,6 +366,27 @@ public final class TypeInfoUtils {
return t;
}
+ private String[] parseParams() {
+ List<String> params = new LinkedList<String>();
+
+ Token t = peek();
+ if (t != null && t.text.equals("(")) {
+ expect("(");
+
+ // checking for null in the for-loop condition prevents null-ptr exception
+ // and allows us to fail more gracefully with a parsing error.
+ for(t = peek(); (t == null) || !t.text.equals(")"); t = expect(",",")")) {
+ params.add(expect("name").text);
+ }
+ if (params.size() == 0) {
+ throw new IllegalArgumentException(
+ "type parameters expected for type string " + typeInfoString);
+ }
+ }
+
+ return params.toArray(new String[params.size()]);
+ }
+
private TypeInfo parseType() {
Token t = expect("type");
@@ -329,7 +396,11 @@ public final class TypeInfoUtils {
.getTypeEntryFromTypeName(t.text);
if (primitiveType != null
&& !primitiveType.primitiveCategory.equals(PrimitiveCategory.UNKNOWN)) {
- return TypeInfoFactory.getPrimitiveTypeInfo(primitiveType.typeName);
+ if (primitiveType.isParameterized()) {
+ primitiveType = primitiveType.addParameters(parseParams());
+ }
+ // If type has qualifiers, the TypeInfo needs them in its type string
+ return TypeInfoFactory.getPrimitiveTypeInfo(primitiveType.toString());
}
// Is this a list type?
@@ -399,6 +470,26 @@ public final class TypeInfoUtils {
+ t.position + " of '" + typeInfoString + "'");
}
+ public PrimitiveParts parsePrimitiveParts() {
+ PrimitiveParts parts = new PrimitiveParts();
+ Token t = expect("type");
+ parts.typeName = t.text;
+ parts.typeParams = parseParams();
+ return parts;
+ }
+ }
+
+ public static class PrimitiveParts {
+ public String typeName;
+ public String[] typeParams;
+ }
+
+ /**
+ * Make some of the TypeInfo parsing available as a utility.
+ */
+ public static PrimitiveParts parsePrimitiveParts(String typeInfoString) {
+ TypeInfoParser parser = new TypeInfoParser(typeInfoString);
+ return parser.parsePrimitiveParts();
}
static Map<TypeInfo, ObjectInspector> cachedStandardObjectInspector =
@@ -414,9 +505,8 @@ public final class TypeInfoUtils {
if (result == null) {
switch (typeInfo.getCategory()) {
case PRIMITIVE: {
- result = PrimitiveObjectInspectorFactory
- .getPrimitiveWritableObjectInspector(((PrimitiveTypeInfo) typeInfo)
- .getPrimitiveCategory());
+ result = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
+ (PrimitiveTypeInfo) typeInfo);
break;
}
case LIST: {
@@ -494,8 +584,7 @@ public final class TypeInfoUtils {
// NOTE: we use JavaPrimitiveObjectInspector instead of
// StandardPrimitiveObjectInspector
result = PrimitiveObjectInspectorFactory
- .getPrimitiveJavaObjectInspector(PrimitiveObjectInspectorUtils
- .getTypeEntryFromTypeName(typeInfo.getTypeName()).primitiveCategory);
+ .getPrimitiveJavaObjectInspector((PrimitiveTypeInfo) typeInfo);
break;
}
case LIST: {
@@ -631,4 +720,42 @@ public final class TypeInfoUtils {
TypeInfoParser parser = new TypeInfoParser(typeString);
return parser.parseTypeInfos().get(0);
}
+
+ /**
+ * Given two types, determine whether conversion needs to occur to compare the two types.
+ * This is needed for cases like varchar, where the TypeInfo for varchar(10) != varchar(5),
+ * but there would be no need to have to convert to compare these values.
+ * @param typeA
+ * @param typeB
+ * @return
+ */
+ public static boolean isConversionRequiredForComparison(TypeInfo typeA, TypeInfo typeB) {
+ if (typeA == typeB) {
+ return false;
+ }
+ if (TypeInfoUtils.doPrimitiveCategoriesMatch(typeA, typeB)) {
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Return the character length of the type
+ * @param typeInfo
+ * @return
+ */
+ public static int getCharacterLengthForType(PrimitiveTypeInfo typeInfo) {
+ switch (typeInfo.getPrimitiveCategory()) {
+ case STRING:
+ return HiveVarchar.MAX_VARCHAR_LENGTH;
+ case VARCHAR:
+ VarcharTypeParams varcharParams = (VarcharTypeParams) typeInfo.getTypeParams();
+ if (varcharParams == null) {
+ throw new RuntimeException("varchar type used without type params");
+ }
+ return varcharParams.getLength();
+ default:
+ return 0;
+ }
+ }
}
Modified: hive/branches/vectorization/service/src/java/org/apache/hive/service/auth/CustomAuthenticationProviderImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/service/src/java/org/apache/hive/service/auth/CustomAuthenticationProviderImpl.java?rev=1524874&r1=1524873&r2=1524874&view=diff
==============================================================================
--- hive/branches/vectorization/service/src/java/org/apache/hive/service/auth/CustomAuthenticationProviderImpl.java (original)
+++ hive/branches/vectorization/service/src/java/org/apache/hive/service/auth/CustomAuthenticationProviderImpl.java Fri Sep 20 00:32:55 2013
@@ -33,7 +33,7 @@ public class CustomAuthenticationProvide
HiveConf conf = new HiveConf();
this.customHandlerClass = (Class<? extends PasswdAuthenticationProvider>)
conf.getClass(
- HiveConf.ConfVars.HIVE_SERVER2_CUSTOM_AUTHENTICATION_CLASS.name(),
+ HiveConf.ConfVars.HIVE_SERVER2_CUSTOM_AUTHENTICATION_CLASS.varname,
PasswdAuthenticationProvider.class);
this.customProvider =
ReflectionUtils.newInstance(this.customHandlerClass, conf);
Modified: hive/branches/vectorization/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java?rev=1524874&r1=1524873&r2=1524874&view=diff
==============================================================================
--- hive/branches/vectorization/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java (original)
+++ hive/branches/vectorization/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java Fri Sep 20 00:32:55 2013
@@ -30,6 +30,7 @@ import java.security.PrivilegedException
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Set;
@@ -37,6 +38,7 @@ import javax.security.auth.Subject;
import javax.security.auth.login.LoginException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -65,6 +67,7 @@ import org.apache.hadoop.mapred.lib.Comb
import org.apache.hadoop.mapred.lib.CombineFileSplit;
import org.apache.hadoop.mapred.lib.TotalOrderPartitioner;
import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UnixUserGroupInformation;
@@ -631,6 +634,38 @@ public class Hadoop20Shims implements Ha
}
@Override
+ public Iterator<FileStatus> listLocatedStatus(final FileSystem fs,
+ final Path path,
+ final PathFilter filter
+ ) throws IOException {
+ return new Iterator<FileStatus>() {
+ private final FileStatus[] result = fs.listStatus(path, filter);
+ private int current = 0;
+
+ @Override
+ public boolean hasNext() {
+ return current < result.length;
+ }
+
+ @Override
+ public FileStatus next() {
+ return result[current++];
+ }
+
+ @Override
+ public void remove() {
+ throw new IllegalArgumentException("Not supported");
+ }
+ };
+ }
+
+ @Override
+ public BlockLocation[] getLocations(FileSystem fs,
+ FileStatus status) throws IOException {
+ return fs.getFileBlockLocations(status, 0, status.getLen());
+ }
+
+ @Override
public boolean isSecurityEnabled() {
return false;
}
@@ -677,6 +712,11 @@ public class Hadoop20Shims implements Ha
}
@Override
+ public TaskAttemptID newTaskAttemptID(JobID jobId, boolean isMap, int taskId, int id) {
+ return new TaskAttemptID(jobId.getJtIdentifier(), jobId.getId(), isMap, taskId, id);
+ }
+
+ @Override
public org.apache.hadoop.mapreduce.JobContext newJobContext(Job job) {
return new org.apache.hadoop.mapreduce.JobContext(job.getConfiguration(), job.getJobID());
}
Modified: hive/branches/vectorization/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java?rev=1524874&r1=1524873&r2=1524874&view=diff
==============================================================================
--- hive/branches/vectorization/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java (original)
+++ hive/branches/vectorization/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java Fri Sep 20 00:32:55 2013
@@ -21,11 +21,15 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.MalformedURLException;
import java.net.URL;
+import java.util.Iterator;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.Trash;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.mapred.JobTracker;
@@ -88,6 +92,11 @@ public class Hadoop20SShims extends Hado
}
@Override
+ public TaskAttemptID newTaskAttemptID(JobID jobId, boolean isMap, int taskId, int id) {
+ return new TaskAttemptID(jobId.getJtIdentifier(), jobId.getId(), isMap, taskId, id);
+ }
+
+ @Override
public org.apache.hadoop.mapreduce.JobContext newJobContext(Job job) {
return new org.apache.hadoop.mapreduce.JobContext(job.getConfiguration(), job.getJobID());
}
@@ -330,4 +339,37 @@ public class Hadoop20SShims extends Hado
public WebHCatJTShim getWebHCatShim(Configuration conf, UserGroupInformation ugi) throws IOException {
return new WebHCatJTShim20S(conf, ugi);//this has state, so can't be cached
}
+
+ @Override
+ public Iterator<FileStatus> listLocatedStatus(final FileSystem fs,
+ final Path path,
+ final PathFilter filter
+ ) throws IOException {
+ return new Iterator<FileStatus>() {
+ private final FileStatus[] result = fs.listStatus(path, filter);
+ private int current = 0;
+
+ @Override
+ public boolean hasNext() {
+ return current < result.length;
+ }
+
+ @Override
+ public FileStatus next() {
+ return result[current++];
+ }
+
+ @Override
+ public void remove() {
+ throw new IllegalArgumentException("Not supported");
+ }
+ };
+ }
+
+ @Override
+ public BlockLocation[] getLocations(FileSystem fs,
+ FileStatus status) throws IOException {
+ return fs.getFileBlockLocations(status, 0, status.getLen());
+ }
+
}
Modified: hive/branches/vectorization/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java?rev=1524874&r1=1524873&r2=1524874&view=diff
==============================================================================
--- hive/branches/vectorization/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java (original)
+++ hive/branches/vectorization/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java Fri Sep 20 00:32:55 2013
@@ -22,12 +22,18 @@ import java.lang.Integer;
import java.net.InetSocketAddress;
import java.net.MalformedURLException;
import java.net.URL;
+import java.util.Iterator;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.Trash;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.mapred.MiniMRCluster;
@@ -100,6 +106,11 @@ public class Hadoop23Shims extends Hadoo
}
@Override
+ public TaskAttemptID newTaskAttemptID(JobID jobId, boolean isMap, int taskId, int id) {
+ return new TaskAttemptID(jobId.getJtIdentifier(), jobId.getId(), isMap ? TaskType.MAP : TaskType.REDUCE, taskId, id);
+ }
+
+ @Override
public org.apache.hadoop.mapreduce.JobContext newJobContext(Job job) {
return new JobContextImpl(job.getConfiguration(), job.getJobID());
}
@@ -339,4 +350,61 @@ public class Hadoop23Shims extends Hadoo
public WebHCatJTShim getWebHCatShim(Configuration conf, UserGroupInformation ugi) throws IOException {
return new WebHCatJTShim23(conf, ugi);//this has state, so can't be cached
}
+
+ @Override
+ public Iterator<FileStatus> listLocatedStatus(final FileSystem fs,
+ final Path path,
+ final PathFilter filter
+ ) throws IOException {
+ return new Iterator<FileStatus>() {
+ private final RemoteIterator<LocatedFileStatus> inner =
+ fs.listLocatedStatus(path);
+ private FileStatus next;
+ {
+ if (inner.hasNext()) {
+ next = inner.next();
+ } else {
+ next = null;
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ return next != null;
+ }
+
+ @Override
+ public FileStatus next() {
+ FileStatus result = next;
+ next = null;
+ try {
+ while (inner.hasNext() && next == null) {
+ next = inner.next();
+ if (filter != null && !filter.accept(next.getPath())) {
+ next = null;
+ }
+ }
+ } catch (IOException ioe) {
+ throw new IllegalArgumentException("Iterator exception", ioe);
+ }
+ return result;
+ }
+
+ @Override
+ public void remove() {
+ throw new IllegalArgumentException("Not supported");
+ }
+ };
+ }
+
+ @Override
+ public BlockLocation[] getLocations(FileSystem fs,
+ FileStatus status) throws IOException {
+ if (status instanceof LocatedFileStatus) {
+ return ((LocatedFileStatus) status).getBlockLocations();
+ } else {
+ return fs.getFileBlockLocations(status, 0, status.getLen());
+ }
+ }
+
}
Modified: hive/branches/vectorization/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java?rev=1524874&r1=1524873&r2=1524874&view=diff
==============================================================================
--- hive/branches/vectorization/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java (original)
+++ hive/branches/vectorization/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java Fri Sep 20 00:32:55 2013
@@ -25,6 +25,7 @@ import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.security.PrivilegedExceptionAction;
+import java.util.Iterator;
import java.util.List;
import javax.security.auth.login.LoginException;
@@ -32,6 +33,7 @@ import javax.security.auth.login.LoginEx
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -316,6 +318,8 @@ public interface HadoopShims {
public TaskAttemptContext newTaskAttemptContext(Configuration conf, final Progressable progressable);
+ public TaskAttemptID newTaskAttemptID(JobID jobId, boolean isMap, int taskId, int id);
+
public JobContext newJobContext(Job job);
/**
@@ -468,6 +472,28 @@ public interface HadoopShims {
Class<RecordReader<K, V>> rrClass) throws IOException;
}
+ /**
+ * Get the block locations for the given directory.
+ * @param fs the file system
+ * @param path the directory name to get the status and block locations
+ * @param filter a filter that needs to accept the file (or null)
+ * @return an iterator for the located file status objects
+ * @throws IOException
+ */
+ Iterator<FileStatus> listLocatedStatus(FileSystem fs, Path path,
+ PathFilter filter) throws IOException;
+
+ /**
+ * For file status returned by listLocatedStatus, convert them into a list
+ * of block locations.
+ * @param fs the file system
+ * @param status the file information
+ * @return the block locations of the file
+ * @throws IOException
+ */
+ BlockLocation[] getLocations(FileSystem fs,
+ FileStatus status) throws IOException;
+
public HCatHadoopShims getHCatShim();
public interface HCatHadoopShims {
Modified: hive/branches/vectorization/testutils/ptest2/src/main/resources/batch-exec.vm
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/testutils/ptest2/src/main/resources/batch-exec.vm?rev=1524874&r1=1524873&r2=1524874&view=diff
==============================================================================
--- hive/branches/vectorization/testutils/ptest2/src/main/resources/batch-exec.vm (original)
+++ hive/branches/vectorization/testutils/ptest2/src/main/resources/batch-exec.vm Fri Sep 20 00:32:55 2013
@@ -32,11 +32,12 @@ export ANT_OPTS="-Xmx1g -XX:MaxPermSize=
export HADOOP_ROOT_LOGGER=INFO,console
export HADOOP_OPTS="-Dhive.log.dir=$logDir -Dhive.query.id=hadoop -Djava.io.tmpdir=$logDir/tmp"
cd $localDir/$instanceName/${repositoryName}-source && \
- timeout 2h ant -Divy.cache.ttl.default=eternal $antArgs \
+ timeout 2h ant test -Dtest.junit.output.format=xml $antArgs \
-Divy.default.ivy.user.dir=$localDir/$instanceName/ivy \
+ -Divy.default.always.check.exact.revision=false \
+ -Divy.cache.ttl.default=eternal -Divy.checkmodified=false \
-Dmvn.local.repo=$localDir/$instanceName/maven \
- -Dtest.junit.output.format=xml test \
- $testArguments 1>$logDir/ant-test.txt 2>&1
+ $antArgs $testArguments 1>$logDir/ant-test.txt 2>&1
ret=$?
if [[ $ret -ne 0 ]]
then
Modified: hive/branches/vectorization/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testBatch.approved.txt
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testBatch.approved.txt?rev=1524874&r1=1524873&r2=1524874&view=diff
==============================================================================
--- hive/branches/vectorization/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testBatch.approved.txt (original)
+++ hive/branches/vectorization/testutils/ptest2/src/test/java/org/apache/hive/ptest/execution/TestScripts.testBatch.approved.txt Fri Sep 20 00:32:55 2013
@@ -31,11 +31,12 @@ export ANT_OPTS="-Xmx1g -XX:MaxPermSize=
export HADOOP_ROOT_LOGGER=INFO,console
export HADOOP_OPTS="-Dhive.log.dir=/some/log/dir -Dhive.query.id=hadoop -Djava.io.tmpdir=/some/log/dir/tmp"
cd /some/local/dir/instance-1/apache-source && \
- timeout 2h ant -Divy.cache.ttl.default=eternal -Dant=arg1 \
+ timeout 2h ant test -Dtest.junit.output.format=xml -Dant=arg1 \
-Divy.default.ivy.user.dir=/some/local/dir/instance-1/ivy \
+ -Divy.default.always.check.exact.revision=false \
+ -Divy.cache.ttl.default=eternal -Divy.checkmodified=false \
-Dmvn.local.repo=/some/local/dir/instance-1/maven \
- -Dtest.junit.output.format=xml test \
- -Dtest=arg1 1>/some/log/dir/ant-test.txt 2>&1
+ -Dant=arg1 -Dtest=arg1 1>/some/log/dir/ant-test.txt 2>&1
ret=$?
if [[ $ret -ne 0 ]]
then