You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/09/25 17:05:01 UTC
svn commit: r1526201 [9/27] - in /hive/branches/vectorization: ./
beeline/src/java/org/apache/hive/beeline/
beeline/src/test/org/apache/hive/beeline/src/test/ bin/ bin/ext/
common/src/java/org/apache/hadoop/hive/conf/
contrib/src/test/results/clientpos...
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1526201&r1=1526200&r2=1526201&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Wed Sep 25 15:04:49 2013
@@ -129,10 +129,7 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.BaseWork;
import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx;
-import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.plan.FileSinkDesc;
import org.apache.hadoop.hive.ql.plan.GroupByDesc;
import org.apache.hadoop.hive.ql.plan.MapWork;
@@ -148,16 +145,10 @@ import org.apache.hadoop.hive.ql.plan.ap
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.stats.StatsFactory;
import org.apache.hadoop.hive.ql.stats.StatsPublisher;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotEqual;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.Serializer;
import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.SequenceFile;
@@ -586,6 +577,31 @@ public final class Utilities {
return null;
}
+ /**
+ * Serializes expression via Kryo.
+ * @param expr Expression.
+ * @return Bytes.
+ */
+ public static byte[] serializeExpressionToKryo(ExprNodeDesc expr) {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ Output output = new Output(baos);
+ runtimeSerializationKryo.get().writeClassAndObject(output, expr);
+ output.close();
+ return baos.toByteArray();
+ }
+
+ /**
+ * Deserializes expression from Kryo.
+ * @param bytes Bytes containing the expression.
+ * @return Expression; null if deserialization succeeded, but the result type is incorrect.
+ */
+ public static ExprNodeDesc deserializeExpressionFromKryo(byte[] bytes) {
+ Input inp = new Input(new ByteArrayInputStream(bytes));
+ Object o = runtimeSerializationKryo.get().readClassAndObject(inp);
+ inp.close();
+ return (o instanceof ExprNodeDesc) ? (ExprNodeDesc)o : null;
+ }
+
public static String serializeExpression(ExprNodeDesc expr) {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
XMLEncoder encoder = new XMLEncoder(baos);
@@ -652,7 +668,6 @@ public final class Utilities {
public void write(Kryo kryo, Output output, java.sql.Date sqlDate) {
output.writeLong(sqlDate.getTime());
}
-
}
private static class CommonTokenSerializer extends com.esotericsoftware.kryo.Serializer<CommonToken> {
@@ -940,17 +955,20 @@ public final class Utilities {
}
public static TableDesc getTableDesc(Table tbl) {
- return (new TableDesc(tbl.getDeserializer().getClass(), tbl.getInputFormatClass(), tbl
- .getOutputFormatClass(), tbl.getMetadata()));
+ Properties props = tbl.getMetadata();
+ props.put(serdeConstants.SERIALIZATION_LIB, tbl.getDeserializer().getClass().getName());
+ return (new TableDesc(tbl.getInputFormatClass(), tbl
+ .getOutputFormatClass(), props));
}
// column names and column types are all delimited by comma
public static TableDesc getTableDesc(String cols, String colTypes) {
- return (new TableDesc(LazySimpleSerDe.class, SequenceFileInputFormat.class,
+ return (new TableDesc(SequenceFileInputFormat.class,
HiveSequenceFileOutputFormat.class, Utilities.makeProperties(
- org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT, "" + Utilities.ctrlaCode,
- org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMNS, cols,
- org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMN_TYPES, colTypes)));
+ serdeConstants.SERIALIZATION_FORMAT, "" + Utilities.ctrlaCode,
+ serdeConstants.LIST_COLUMNS, cols,
+ serdeConstants.LIST_COLUMN_TYPES, colTypes,
+ serdeConstants.SERIALIZATION_LIB,LazySimpleSerDe.class.getName())));
}
public static PartitionDesc getPartitionDesc(Partition part) throws HiveException {
@@ -2367,126 +2385,6 @@ public final class Utilities {
}
/**
- * Check if a function can be pushed down to JDO.
- * Now only {compares, AND, OR} are supported.
- * @param func a generic function.
- * @return true if this function can be pushed down to JDO filter.
- */
- private static boolean supportedJDOFuncs(GenericUDF func) {
- // TODO: we might also want to add "not" and "between" here in future.
- // TODO: change to GenericUDFBaseCompare once DN is upgraded
- // (see HIVE-2609 - in DN 2.0, substrings do not work in MySQL).
- return func instanceof GenericUDFOPEqual ||
- func instanceof GenericUDFOPNotEqual ||
- func instanceof GenericUDFOPAnd ||
- func instanceof GenericUDFOPOr;
- }
-
- /**
- * Check if a function can be pushed down to JDO for integral types.
- * Only {=, !=} are supported. lt/gt/etc. to be dealt with in HIVE-4888.
- * @param func a generic function.
- * @return true iff this function can be pushed down to JDO filter for integral types.
- */
- private static boolean doesJDOFuncSupportIntegral(GenericUDF func) {
- // AND, OR etc. don't need to be specified here.
- return func instanceof GenericUDFOPEqual ||
- func instanceof GenericUDFOPNotEqual;
- }
-
- /**
- * @param type type
- * @param constant The constant, if any.
- * @return true iff type is an integral type.
- */
- private static boolean isIntegralType(String type) {
- return type.equals(serdeConstants.TINYINT_TYPE_NAME) ||
- type.equals(serdeConstants.SMALLINT_TYPE_NAME) ||
- type.equals(serdeConstants.INT_TYPE_NAME) ||
- type.equals(serdeConstants.BIGINT_TYPE_NAME);
- }
-
- /**
- * Check if the partition pruning expression can be pushed down to JDO filtering.
- * The partition expression contains only partition columns.
- * The criteria that an expression can be pushed down are that:
- * 1) the expression only contains function specified in supportedJDOFuncs().
- * Now only {=, AND, OR} can be pushed down.
- * 2) the partition column type and the constant type have to be String. This is
- * restriction by the current JDO filtering implementation.
- * @param tab The table that contains the partition columns.
- * @param expr the partition pruning expression
- * @param parent parent UDF of expr if parent exists and contains a UDF; otherwise null.
- * @return null if the partition pruning expression can be pushed down to JDO filtering.
- */
- public static String checkJDOPushDown(
- Table tab, ExprNodeDesc expr, GenericUDF parent) {
- boolean isConst = expr instanceof ExprNodeConstantDesc;
- boolean isCol = !isConst && (expr instanceof ExprNodeColumnDesc);
- boolean isIntegralSupported = (parent != null) && (isConst || isCol)
- && doesJDOFuncSupportIntegral(parent);
-
- // JDO filter now only support String typed literals, as well as integers
- // for some operators; see Filter.g and ExpressionTree.java.
- if (isConst) {
- Object value = ((ExprNodeConstantDesc)expr).getValue();
- if (value instanceof String) {
- return null;
- }
- if (isIntegralSupported && isIntegralType(expr.getTypeInfo().getTypeName())) {
- return null;
- }
- return "Constant " + value + " is not string "
- + (isIntegralSupported ? "or integral ": "") + "type: " + expr.getTypeInfo().getTypeName();
- } else if (isCol) {
- TypeInfo type = expr.getTypeInfo();
- if (type.getTypeName().equals(serdeConstants.STRING_TYPE_NAME)
- || (isIntegralSupported && isIntegralType(type.getTypeName()))) {
- String colName = ((ExprNodeColumnDesc)expr).getColumn();
- for (FieldSchema fs: tab.getPartCols()) {
- if (fs.getName().equals(colName)) {
- if (fs.getType().equals(serdeConstants.STRING_TYPE_NAME)
- || (isIntegralSupported && isIntegralType(fs.getType()))) {
- return null;
- }
- return "Partition column " + fs.getName() + " is not string "
- + (isIntegralSupported ? "or integral ": "") + "type: " + fs.getType();
- }
- }
- assert(false); // cannot find the partition column!
- } else {
- return "Column " + expr.getExprString() + " is not string "
- + (isIntegralSupported ? "or integral ": "") + "type: " + type.getTypeName();
- }
- } else if (expr instanceof ExprNodeGenericFuncDesc) {
- ExprNodeGenericFuncDesc funcDesc = (ExprNodeGenericFuncDesc) expr;
- GenericUDF func = funcDesc.getGenericUDF();
- if (!supportedJDOFuncs(func)) {
- return "Expression " + expr.getExprString() + " cannot be evaluated";
- }
- boolean allChildrenConstant = true;
- List<ExprNodeDesc> children = funcDesc.getChildExprs();
- for (ExprNodeDesc child: children) {
- if (!(child instanceof ExprNodeConstantDesc)) {
- allChildrenConstant = false;
- }
- String message = checkJDOPushDown(tab, child, func);
- if (message != null) {
- return message;
- }
- }
-
- // If all the children of the expression are constants then JDO cannot parse the expression
- // see Filter.g
- if (allChildrenConstant) {
- return "Expression " + expr.getExprString() + " has only constants as children.";
- }
- return null;
- }
- return "Expression " + expr.getExprString() + " cannot be evaluated";
- }
-
- /**
* The check here is kind of not clean. It first use a for loop to go through
* all input formats, and choose the ones that extend ReworkMapredInputFormat
* to a set. And finally go through the ReworkMapredInputFormat set, and call
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/PTFRowContainer.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/PTFRowContainer.java?rev=1526201&r1=1526200&r2=1526201&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/PTFRowContainer.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/PTFRowContainer.java Wed Sep 25 15:04:49 2013
@@ -34,6 +34,7 @@ import org.apache.hadoop.hive.ql.io.Hive
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.PTFDeserializer;
import org.apache.hadoop.hive.ql.plan.TableDesc;
+import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.io.BytesWritable;
@@ -226,17 +227,15 @@ public class PTFRowContainer<Row extends
public static TableDesc createTableDesc(StructObjectInspector oI) {
Map<String,String> props = new HashMap<String,String>();
PTFDeserializer.addOIPropertiestoSerDePropsMap(oI, props);
- String colNames = props.get(org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMNS);
- String colTypes = props.get(org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMN_TYPES);
- TableDesc tblDesc = new TableDesc(LazyBinarySerDe.class,
+ String colNames = props.get(serdeConstants.LIST_COLUMNS);
+ String colTypes = props.get(serdeConstants.LIST_COLUMN_TYPES);
+ TableDesc tblDesc = new TableDesc(
PTFSequenceFileInputFormat.class, PTFHiveSequenceFileOutputFormat.class,
Utilities.makeProperties(
- org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT, ""
- + Utilities.ctrlaCode,
- org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMNS, colNames
- .toString(),
- org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMN_TYPES,
- colTypes.toString()));
+ serdeConstants.SERIALIZATION_FORMAT, ""+ Utilities.ctrlaCode,
+ serdeConstants.LIST_COLUMNS, colNames.toString(),
+ serdeConstants.LIST_COLUMN_TYPES,colTypes.toString(),
+ serdeConstants.SERIALIZATION_LIB,LazyBinarySerDe.class.getName()));
return tblDesc;
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java?rev=1526201&r1=1526200&r2=1526201&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java Wed Sep 25 15:04:49 2013
@@ -124,28 +124,17 @@ public class VectorizedRowBatchCtx {
PartitionDesc part = HiveFileFormatUtils
.getPartitionDescFromPathRecursively(pathToPartitionInfo,
split.getPath(), IOPrepareCache.get().getPartitionDescMap());
- Class serdeclass = part.getDeserializerClass();
String partitionPath = split.getPath().getParent().toString();
columnTypeMap = Utilities
.getMapRedWork(hiveConf).getMapWork().getScratchColumnVectorTypes()
.get(partitionPath);
- if (serdeclass == null) {
- String className = part.getSerdeClassName();
- if ((className == null) || (className.isEmpty())) {
- throw new HiveException(
- "SerDe class or the SerDe class name is not set for table: "
- + part.getProperties().getProperty("name"));
- }
- serdeclass = hiveConf.getClassByName(className);
- }
-
Properties partProps =
(part.getPartSpec() == null || part.getPartSpec().isEmpty()) ?
part.getTableDesc().getProperties() : part.getProperties();
- Deserializer partDeserializer = (Deserializer) serdeclass.newInstance();
+ Deserializer partDeserializer = part.getDeserializer();
partDeserializer.initialize(hiveConf, partProps);
StructObjectInspector partRawRowObjectInspector = (StructObjectInspector) partDeserializer
.getObjectInspector();
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java?rev=1526201&r1=1526200&r2=1526201&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java Wed Sep 25 15:04:49 2013
@@ -307,8 +307,8 @@ public class CombineHiveInputFormat<K ex
Class inputFormatClass = part.getInputFileFormatClass();
String inputFormatClassName = inputFormatClass.getName();
InputFormat inputFormat = getInputFormatFromCache(inputFormatClass, job);
- String deserializerClassName = part.getDeserializerClass() == null ? null
- : part.getDeserializerClass().getName();
+ String deserializerClassName = part.getDeserializer() == null ? null
+ : part.getDeserializer().getClass().getName();
// Since there is no easy way of knowing whether MAPREDUCE-1597 is present in the tree or not,
// we use a configuration variable for the same
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java?rev=1526201&r1=1526200&r2=1526201&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java Wed Sep 25 15:04:49 2013
@@ -49,6 +49,7 @@ import org.apache.hadoop.fs.FsShell;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.HiveMetaException;
import org.apache.hadoop.hive.metastore.HiveMetaHook;
import org.apache.hadoop.hive.metastore.HiveMetaHookLoader;
@@ -80,6 +81,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.index.HiveIndexHandler;
import org.apache.hadoop.hive.ql.optimizer.listbucketingpruner.ListBucketingPrunerUtils;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.session.CreateTableAutomaticGrant;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.serde2.Deserializer;
@@ -1899,13 +1901,43 @@ private void constructOneLBLocationMap(F
List<org.apache.hadoop.hive.metastore.api.Partition> tParts = getMSC().listPartitionsByFilter(
tbl.getDbName(), tbl.getTableName(), filter, (short)-1);
- List<Partition> results = new ArrayList<Partition>(tParts.size());
+ return convertFromMetastore(tbl, tParts, null);
+ }
- for (org.apache.hadoop.hive.metastore.api.Partition tPart: tParts) {
- Partition part = new Partition(tbl, tPart);
- results.add(part);
+ private static List<Partition> convertFromMetastore(Table tbl,
+ List<org.apache.hadoop.hive.metastore.api.Partition> src,
+ List<Partition> dest) throws HiveException {
+ if (src == null) {
+ return dest;
+ }
+ if (dest == null) {
+ dest = new ArrayList<Partition>(src.size());
}
- return results;
+ for (org.apache.hadoop.hive.metastore.api.Partition tPart : src) {
+ dest.add(new Partition(tbl, tPart));
+ }
+ return dest;
+ }
+
+ /**
+ * Get a list of Partitions by expr.
+ * @param tbl The table containing the partitions.
+ * @param expr A serialized expression for partition predicates.
+ * @param conf Hive config.
+ * @param result the resulting list of partitions
+ * @return whether the resulting list contains partitions which may or may not match the expr
+ */
+ public boolean getPartitionsByExpr(Table tbl, ExprNodeDesc expr, HiveConf conf,
+ List<Partition> result) throws HiveException, TException {
+ assert result != null;
+ byte[] exprBytes = Utilities.serializeExpressionToKryo(expr);
+ String defaultPartitionName = HiveConf.getVar(conf, ConfVars.DEFAULTPARTITIONNAME);
+ List<org.apache.hadoop.hive.metastore.api.Partition> msParts =
+ new ArrayList<org.apache.hadoop.hive.metastore.api.Partition>();
+ boolean hasUnknownParts = getMSC().listPartitionsByExpr(tbl.getDbName(),
+ tbl.getTableName(), exprBytes, defaultPartitionName, (short)-1, msParts);
+ convertFromMetastore(tbl, msParts, result);
+ return hasUnknownParts;
}
public void validatePartitionNameCharacters(List<String> partVals) throws HiveException {
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MetadataOnlyOptimizer.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MetadataOnlyOptimizer.java?rev=1526201&r1=1526200&r2=1526201&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MetadataOnlyOptimizer.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MetadataOnlyOptimizer.java Wed Sep 25 15:04:49 2013
@@ -54,6 +54,7 @@ import org.apache.hadoop.hive.ql.plan.Ma
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.PartitionDesc;
import org.apache.hadoop.hive.ql.plan.TableScanDesc;
+import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.NullStructSerDe;
/**
@@ -210,8 +211,8 @@ public class MetadataOnlyOptimizer imple
if (desc != null) {
desc.setInputFileFormatClass(OneNullRowInputFormat.class);
desc.setOutputFileFormatClass(HiveIgnoreKeyTextOutputFormat.class);
- desc.setDeserializerClass(NullStructSerDe.class);
- desc.setSerdeClassName(NullStructSerDe.class.getName());
+ desc.getProperties().setProperty(serdeConstants.SERIALIZATION_LIB,
+ NullStructSerDe.class.getName());
}
return desc;
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java?rev=1526201&r1=1526200&r2=1526201&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java Wed Sep 25 15:04:49 2013
@@ -32,6 +32,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.common.ObjectPair;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
@@ -241,6 +242,19 @@ public class PartitionPruner implements
return expr;
}
+ /**
+ * @param expr Expression.
+ * @return True iff expr contains any non-native user-defined functions.
+ */
+ static private boolean hasUserFunctions(ExprNodeDesc expr) {
+ if (!(expr instanceof ExprNodeGenericFuncDesc)) return false;
+ if (!FunctionRegistry.isNativeFuncExpr((ExprNodeGenericFuncDesc)expr)) return true;
+ for (ExprNodeDesc child : expr.getChildren()) {
+ if (hasUserFunctions(child)) return true;
+ }
+ return false;
+ }
+
private static PrunedPartitionList getPartitionsFromServer(Table tab,
ExprNodeDesc prunerExpr, HiveConf conf, String alias) throws HiveException {
try {
@@ -258,36 +272,55 @@ public class PartitionPruner implements
}
if (prunerExpr == null) {
- // This can happen when hive.mapred.mode=nonstrict and there is no predicates at all.
+ // Non-strict mode, and there is no predicates at all - get everything.
return new PrunedPartitionList(tab, Hive.get().getAllPartitionsForPruner(tab), false);
}
- // Remove virtual columns. See javadoc for details.
+ // Replace virtual columns with nulls. See javadoc for details.
prunerExpr = removeNonPartCols(prunerExpr, extractPartColNames(tab));
- // Remove all unknown parts e.g. non-partition columns. See javadoc for details.
+ // Remove all parts that are not partition columns. See javadoc for details.
ExprNodeDesc compactExpr = compactExpr(prunerExpr.clone());
String oldFilter = prunerExpr.getExprString();
if (compactExpr == null) {
- // This could happen when hive.mapred.mode=nonstrict and all the predicates
- // are on non-partition columns.
+ // Non-strict mode, and all the predicates are on non-partition columns - get everything.
LOG.debug("Filter " + oldFilter + " was null after compacting");
return new PrunedPartitionList(tab, Hive.get().getAllPartitionsForPruner(tab), true);
}
- Set<Partition> partitions = new LinkedHashSet<Partition>();
+ LOG.debug("Filter w/ compacting: " + compactExpr.getExprString()
+ + "; filter w/o compacting: " + oldFilter);
+
+ // Finally, check the filter for non-built-in UDFs. If these are present, we cannot
+ // do filtering on the server, and have to fall back to client path.
+ boolean doEvalClientSide = hasUserFunctions(compactExpr);
+
+ // Now filter.
+ List<Partition> partitions = new ArrayList<Partition>();
boolean hasUnknownPartitions = false;
- String message = Utilities.checkJDOPushDown(tab, compactExpr, null);
- if (message != null) {
- LOG.info(ErrorMsg.INVALID_JDO_FILTER_EXPRESSION.getMsg("by condition '"
- + message + "'"));
- hasUnknownPartitions = pruneBySequentialScan(tab, partitions, prunerExpr, conf);
- } else {
- String filter = compactExpr.getExprString();
- LOG.debug("Filter w/ compacting: " + filter +"; filter w/o compacting: " + oldFilter);
- hasUnknownPartitions = !filter.equals(oldFilter);
- partitions.addAll(Hive.get().getPartitionsByFilter(tab, filter));
+ PerfLogger perfLogger = PerfLogger.getPerfLogger();
+ if (!doEvalClientSide) {
+ perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.PARTITION_RETRIEVING);
+ try {
+ hasUnknownPartitions = Hive.get().getPartitionsByExpr(
+ tab, compactExpr, conf, partitions);
+ } catch (IMetaStoreClient.IncompatibleMetastoreException ime) {
+ // TODO: backward compat for Hive <= 0.12. Can be removed later.
+ LOG.warn("Metastore doesn't support getPartitionsByExpr", ime);
+ doEvalClientSide = true;
+ } finally {
+ perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.PARTITION_RETRIEVING);
+ }
}
- return new PrunedPartitionList(tab, partitions, hasUnknownPartitions);
+ if (doEvalClientSide) {
+ // Either we have user functions, or metastore is old version - filter names locally.
+ hasUnknownPartitions = pruneBySequentialScan(tab, partitions, compactExpr, conf);
+ }
+ // The partitions are "unknown" if the call says so due to the expression
+ // evaluator returning null for a partition, or if we sent a partial expression to
+ // metastore and so some partitions may have no data based on other filters.
+ boolean isPruningByExactFilter = oldFilter.equals(compactExpr.getExprString());
+ return new PrunedPartitionList(tab, new LinkedHashSet<Partition>(partitions),
+ hasUnknownPartitions || !isPruningByExactFilter);
} catch (HiveException e) {
throw e;
} catch (Exception e) {
@@ -297,15 +330,15 @@ public class PartitionPruner implements
/**
* Pruning partition by getting the partition names first and pruning using Hive expression
- * evaluator.
+ * evaluator on client.
* @param tab the table containing the partitions.
* @param partitions the resulting partitions.
* @param prunerExpr the SQL predicate that involves partition columns.
* @param conf Hive Configuration object, can not be NULL.
* @return true iff the partition pruning expression contains non-partition columns.
*/
- static private boolean pruneBySequentialScan(Table tab, Set<Partition> partitions,
- ExprNodeDesc prunerExpr, HiveConf conf) throws Exception {
+ static private boolean pruneBySequentialScan(Table tab, List<Partition> partitions,
+ ExprNodeDesc prunerExpr, HiveConf conf) throws HiveException, MetaException {
PerfLogger perfLogger = PerfLogger.getPerfLogger();
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.PRUNE_LISTING);
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java?rev=1526201&r1=1526200&r2=1526201&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java Wed Sep 25 15:04:49 2013
@@ -1915,10 +1915,9 @@ public class DDLSemanticAnalyzer extends
String[] colTypes = schema.split("#");
prop.setProperty("columns", colTypes[0]);
prop.setProperty("columns.types", colTypes[1]);
-
+ prop.setProperty(serdeConstants.SERIALIZATION_LIB, LazySimpleSerDe.class.getName());
FetchWork fetch = new FetchWork(ctx.getResFile().toString(), new TableDesc(
- LazySimpleSerDe.class, TextInputFormat.class,
- IgnoreKeyTextOutputFormat.class, prop), -1);
+ TextInputFormat.class,IgnoreKeyTextOutputFormat.class, prop), -1);
fetch.setSerializationNullFormat(" ");
return (FetchTask) TaskFactory.get(fetch, conf);
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/AggregationDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/AggregationDesc.java?rev=1526201&r1=1526200&r2=1526201&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/AggregationDesc.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/AggregationDesc.java Wed Sep 25 15:04:49 2013
@@ -18,28 +18,38 @@
package org.apache.hadoop.hive.ql.plan;
+import java.io.Externalizable;
+import java.io.Serializable;
+
+import org.apache.hadoop.hive.common.JavaUtils;
+import org.apache.hadoop.hive.ql.exec.PTFUtils;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
+import org.apache.hadoop.util.ReflectionUtils;
/**
* AggregationDesc.
*
*/
public class AggregationDesc implements java.io.Serializable {
+
+ static {
+ PTFUtils.makeTransient(AggregationDesc.class, "genericUDAFEvaluator");
+ }
+
private static final long serialVersionUID = 1L;
private String genericUDAFName;
- /**
- * In case genericUDAFEvaluator is Serializable, we will serialize the object.
- *
- * In case genericUDAFEvaluator does not implement Serializable, Java will
- * remember the class of genericUDAFEvaluator and creates a new instance when
- * deserialized. This is exactly what we want.
- */
- private GenericUDAFEvaluator genericUDAFEvaluator;
private java.util.ArrayList<ExprNodeDesc> parameters;
private boolean distinct;
private GenericUDAFEvaluator.Mode mode;
+ // used for GenericUDAFEvaluator
+ private String genericUDAFEvaluatorClassName;
+ // used for GenericUDAFBridgeEvaluator
+ private GenericUDAFEvaluator genericUDAFWritableEvaluator;
+
+ private transient GenericUDAFEvaluator genericUDAFEvaluator;
+
public AggregationDesc() {
}
@@ -48,10 +58,10 @@ public class AggregationDesc implements
final java.util.ArrayList<ExprNodeDesc> parameters,
final boolean distinct, final GenericUDAFEvaluator.Mode mode) {
this.genericUDAFName = genericUDAFName;
- this.genericUDAFEvaluator = genericUDAFEvaluator;
this.parameters = parameters;
this.distinct = distinct;
this.mode = mode;
+ setGenericUDAFEvaluator(genericUDAFEvaluator);
}
public void setGenericUDAFName(final String genericUDAFName) {
@@ -65,10 +75,44 @@ public class AggregationDesc implements
public void setGenericUDAFEvaluator(
final GenericUDAFEvaluator genericUDAFEvaluator) {
this.genericUDAFEvaluator = genericUDAFEvaluator;
+ if (genericUDAFEvaluator instanceof Serializable ||
+ genericUDAFEvaluator instanceof Externalizable) {
+ this.genericUDAFWritableEvaluator = genericUDAFEvaluator;
+ } else {
+ this.genericUDAFEvaluatorClassName = genericUDAFEvaluator.getClass().getName();
+ }
}
public GenericUDAFEvaluator getGenericUDAFEvaluator() {
- return genericUDAFEvaluator;
+ if (genericUDAFEvaluator != null) {
+ return genericUDAFEvaluator;
+ }
+ if (genericUDAFWritableEvaluator != null) {
+ return genericUDAFEvaluator = genericUDAFWritableEvaluator;
+ }
+ try {
+ return genericUDAFEvaluator =
+ ReflectionUtils.newInstance(Class.forName(genericUDAFEvaluatorClassName, true,
+ JavaUtils.getClassLoader()).asSubclass(GenericUDAFEvaluator.class), null);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public String getGenericUDAFEvaluatorClassName() {
+ return genericUDAFEvaluatorClassName;
+ }
+
+ public void setGenericUDAFEvaluatorClassName(String genericUDAFEvaluatorClassName) {
+ this.genericUDAFEvaluatorClassName = genericUDAFEvaluatorClassName;
+ }
+
+ public GenericUDAFEvaluator getGenericUDAFWritableEvaluator() {
+ return genericUDAFWritableEvaluator;
+ }
+
+ public void setGenericUDAFWritableEvaluator(GenericUDAFEvaluator genericUDAFWritableEvaluator) {
+ this.genericUDAFWritableEvaluator = genericUDAFWritableEvaluator;
}
public java.util.ArrayList<ExprNodeDesc> getParameters() {
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java?rev=1526201&r1=1526200&r2=1526201&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java Wed Sep 25 15:04:49 2013
@@ -47,8 +47,6 @@ import org.apache.hadoop.hive.ql.udf.ptf
import org.apache.hadoop.hive.ql.udf.ptf.TableFunctionResolver;
import org.apache.hadoop.hive.ql.udf.ptf.WindowingTableFunction.WindowingTableFunctionResolver;
import org.apache.hadoop.hive.serde2.SerDe;
-import org.apache.hadoop.hive.serde2.SerDeException;
-import org.apache.hadoop.hive.serde2.SerDeUtils;
import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
@@ -259,13 +257,13 @@ public class PTFDeserializer {
serDeProps.setProperty(serdeName, serdePropsMap.get(serdeName));
}
try {
- SerDe serDe = (SerDe) SerDeUtils.lookupDeserializer(serdeClassName);
+ SerDe serDe = ReflectionUtils.newInstance(hConf.getClassByName(serdeClassName).
+ asSubclass(SerDe.class), hConf);
serDe.initialize(hConf, serDeProps);
shp.setSerde(serDe);
StructObjectInspector outOI = PTFPartition.setupPartitionOutputOI(serDe, OI);
shp.setOI(outOI);
- } catch (SerDeException se)
- {
+ } catch (Exception se) {
throw new HiveException(se);
}
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java?rev=1526201&r1=1526200&r2=1526201&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java Wed Sep 25 15:04:49 2013
@@ -20,16 +20,20 @@ package org.apache.hadoop.hive.ql.plan;
import java.io.Serializable;
import java.util.Enumeration;
+import java.util.LinkedHashMap;
import java.util.Properties;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
+import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.Deserializer;
-import org.apache.hadoop.hive.serde2.SerDeException;
-import org.apache.hadoop.hive.serde2.SerDeUtils;
import org.apache.hadoop.mapred.InputFormat;
/**
@@ -40,12 +44,10 @@ import org.apache.hadoop.mapred.InputFor
public class PartitionDesc implements Serializable, Cloneable {
private static final long serialVersionUID = 2L;
private TableDesc tableDesc;
- private java.util.LinkedHashMap<String, String> partSpec;
- private java.lang.Class<? extends org.apache.hadoop.hive.serde2.Deserializer> deserializerClass;
+ private LinkedHashMap<String, String> partSpec;
private Class<? extends InputFormat> inputFileFormatClass;
private Class<? extends HiveOutputFormat> outputFileFormatClass;
- private java.util.Properties properties;
- private String serdeClassName;
+ private Properties properties;
private String baseFileName;
@@ -56,65 +58,25 @@ public class PartitionDesc implements Se
public PartitionDesc() {
}
- public PartitionDesc(final TableDesc table,
- final java.util.LinkedHashMap<String, String> partSpec) {
- this(table, partSpec, null, null, null, null, null);
- }
-
- public PartitionDesc(final TableDesc table,
- final java.util.LinkedHashMap<String, String> partSpec,
- final Class<? extends Deserializer> serdeClass,
- final Class<? extends InputFormat> inputFileFormatClass,
- final Class<?> outputFormat, final java.util.Properties properties,
- final String serdeClassName) {
+ public PartitionDesc(final TableDesc table, final LinkedHashMap<String, String> partSpec) {
this.tableDesc = table;
- this.properties = properties;
this.partSpec = partSpec;
- deserializerClass = serdeClass;
- this.inputFileFormatClass = inputFileFormatClass;
- if (outputFormat != null) {
- outputFileFormatClass = HiveFileFormatUtils
- .getOutputFormatSubstitute(outputFormat,false);
- }
- if (serdeClassName != null) {
- this.serdeClassName = serdeClassName;
- } else if (properties != null) {
- this.serdeClassName = properties
- .getProperty(org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_LIB);
- }
}
- public PartitionDesc(final org.apache.hadoop.hive.ql.metadata.Partition part)
- throws HiveException {
+ public PartitionDesc(final Partition part) throws HiveException {
tableDesc = Utilities.getTableDesc(part.getTable());
properties = part.getMetadataFromPartitionSchema();
partSpec = part.getSpec();
- deserializerClass = part.getDeserializer(properties).getClass();
inputFileFormatClass = part.getInputFormatClass();
outputFileFormatClass = part.getOutputFormatClass();
- serdeClassName = properties
- .getProperty(org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_LIB);
- ;
}
- public PartitionDesc(final org.apache.hadoop.hive.ql.metadata.Partition part,
- final TableDesc tblDesc) throws HiveException {
+ public PartitionDesc(final Partition part,final TableDesc tblDesc) throws HiveException {
tableDesc = tblDesc;
properties = part.getSchemaFromTableSchema(tblDesc.getProperties()); // each partition maintains a large properties
partSpec = part.getSpec();
- // deserializerClass = part.getDeserializer(properties).getClass();
- Deserializer deserializer;
- try {
- deserializer = SerDeUtils.lookupDeserializer(
- properties.getProperty(org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_LIB));
- } catch (SerDeException e) {
- throw new HiveException(e);
- }
- deserializerClass = deserializer.getClass();
inputFileFormatClass = part.getInputFormatClass();
outputFileFormatClass = part.getOutputFormatClass();
- serdeClassName = properties.getProperty(
- org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_LIB);
}
@Explain(displayName = "")
@@ -127,27 +89,15 @@ public class PartitionDesc implements Se
}
@Explain(displayName = "partition values")
- public java.util.LinkedHashMap<String, String> getPartSpec() {
+ public LinkedHashMap<String, String> getPartSpec() {
return partSpec;
}
- public void setPartSpec(final java.util.LinkedHashMap<String, String> partSpec) {
+ public void setPartSpec(final LinkedHashMap<String, String> partSpec) {
this.partSpec = partSpec;
}
- public java.lang.Class<? extends org.apache.hadoop.hive.serde2.Deserializer> getDeserializerClass() {
- if (deserializerClass == null && tableDesc != null) {
- setDeserializerClass(tableDesc.getDeserializerClass());
- }
- return deserializerClass;
- }
-
- public void setDeserializerClass(
- final java.lang.Class<? extends org.apache.hadoop.hive.serde2.Deserializer> serdeClass) {
- deserializerClass = serdeClass;
- }
-
- public Class<? extends InputFormat> getInputFileFormatClass() {
+ public Class<? extends InputFormat> getInputFileFormatClass() {
if (inputFileFormatClass == null && tableDesc != null) {
setInputFileFormatClass(tableDesc.getInputFileFormatClass());
}
@@ -157,10 +107,12 @@ public class PartitionDesc implements Se
/**
* Return a deserializer object corresponding to the tableDesc.
*/
- public Deserializer getDeserializer() throws Exception {
- Deserializer de = deserializerClass.newInstance();
- de.initialize(null, properties);
- return de;
+ public Deserializer getDeserializer() {
+ try {
+ return MetaStoreUtils.getDeserializer(Hive.get().getConf(), getProperties());
+ } catch (Exception e) {
+ return null;
+ }
}
public void setInputFileFormatClass(
@@ -181,14 +133,14 @@ public class PartitionDesc implements Se
}
@Explain(displayName = "properties", normalExplain = false)
- public java.util.Properties getProperties() {
+ public Properties getProperties() {
if (properties == null && tableDesc != null) {
return tableDesc.getProperties();
}
return properties;
}
- public java.util.Properties getOverlayedProperties(){
+ public Properties getOverlayedProperties(){
if (tableDesc != null) {
Properties overlayedProps = new Properties(tableDesc.getProperties());
overlayedProps.putAll(getProperties());
@@ -198,7 +150,7 @@ public class PartitionDesc implements Se
}
}
- public void setProperties(final java.util.Properties properties) {
+ public void setProperties(final Properties properties) {
this.properties = properties;
}
@@ -207,24 +159,12 @@ public class PartitionDesc implements Se
*/
@Explain(displayName = "serde")
public String getSerdeClassName() {
- if (serdeClassName == null && tableDesc != null) {
- setSerdeClassName(tableDesc.getSerdeClassName());
- }
- return serdeClassName;
- }
-
- /**
- * @param serdeClassName
- * the serde Class Name to set
- */
- public void setSerdeClassName(String serdeClassName) {
- this.serdeClassName = serdeClassName;
+ return getProperties().getProperty(serdeConstants.SERIALIZATION_LIB);
}
@Explain(displayName = "name")
public String getTableName() {
- return getProperties().getProperty(
- org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_NAME);
+ return getProperties().getProperty(hive_metastoreConstants.META_TABLE_NAME);
}
@Explain(displayName = "input format")
@@ -250,8 +190,6 @@ public class PartitionDesc implements Se
public PartitionDesc clone() {
PartitionDesc ret = new PartitionDesc();
- ret.setSerdeClassName(serdeClassName);
- ret.setDeserializerClass(deserializerClass);
ret.inputFileFormatClass = inputFileFormatClass;
ret.outputFileFormatClass = outputFileFormatClass;
if (properties != null) {
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java?rev=1526201&r1=1526200&r2=1526201&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java Wed Sep 25 15:04:49 2013
@@ -18,7 +18,6 @@
package org.apache.hadoop.hive.ql.plan;
-import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@@ -132,12 +131,9 @@ public final class PlanUtils {
serdeConstants.ESCAPE_CHAR, localDirectoryDesc.getFieldEscape());
}
if (localDirectoryDesc.getSerName() != null) {
- tableDesc.setSerdeClassName(localDirectoryDesc.getSerName());
tableDesc.getProperties().setProperty(
serdeConstants.SERIALIZATION_LIB, localDirectoryDesc.getSerName());
- tableDesc.setDeserializerClass(
- (Class<? extends Deserializer>) Class.forName(localDirectoryDesc.getSerName()));
- }
+ }
if (localDirectoryDesc.getOutputFormat() != null){
tableDesc.setOutputFileFormatClass(Class.forName(localDirectoryDesc.getOutputFormat()));
}
@@ -263,7 +259,8 @@ public final class PlanUtils {
inputFormat = TextInputFormat.class;
outputFormat = IgnoreKeyTextOutputFormat.class;
}
- return new TableDesc(serdeClass, inputFormat, outputFormat, properties);
+ properties.setProperty(serdeConstants.SERIALIZATION_LIB, serdeClass.getName());
+ return new TableDesc(inputFormat, outputFormat, properties);
}
public static TableDesc getDefaultQueryOutputTableDesc(String cols, String colTypes,
@@ -274,7 +271,7 @@ public final class PlanUtils {
tblDesc.getProperties().setProperty(serdeConstants.ESCAPE_CHAR, "\\");
//enable extended nesting levels
tblDesc.getProperties().setProperty(
- LazySimpleSerDe.SERIALIZATION_EXTEND_NESTING_LEVELS, "true");
+ LazySimpleSerDe.SERIALIZATION_EXTEND_NESTING_LEVELS, "true");
return tblDesc;
}
@@ -354,11 +351,11 @@ public final class PlanUtils {
* "array<string>".
*/
public static TableDesc getDefaultTableDesc(String separatorCode) {
- return new TableDesc(MetadataTypedColumnsetSerDe.class,
+ return new TableDesc(
TextInputFormat.class, IgnoreKeyTextOutputFormat.class, Utilities
.makeProperties(
- org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT,
- separatorCode));
+ org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT,separatorCode,
+ serdeConstants.SERIALIZATION_LIB,MetadataTypedColumnsetSerDe.class.getName()));
}
/**
@@ -366,25 +363,27 @@ public final class PlanUtils {
*/
public static TableDesc getReduceKeyTableDesc(List<FieldSchema> fieldSchemas,
String order) {
- return new TableDesc(BinarySortableSerDe.class,
+ return new TableDesc(
SequenceFileInputFormat.class, SequenceFileOutputFormat.class,
Utilities.makeProperties(serdeConstants.LIST_COLUMNS, MetaStoreUtils
.getColumnNamesFromFieldSchema(fieldSchemas),
serdeConstants.LIST_COLUMN_TYPES, MetaStoreUtils
.getColumnTypesFromFieldSchema(fieldSchemas),
- serdeConstants.SERIALIZATION_SORT_ORDER, order));
+ serdeConstants.SERIALIZATION_SORT_ORDER, order,
+ serdeConstants.SERIALIZATION_LIB, BinarySortableSerDe.class.getName()));
}
/**
* Generate the table descriptor for Map-side join key.
*/
public static TableDesc getMapJoinKeyTableDesc(List<FieldSchema> fieldSchemas) {
- return new TableDesc(LazyBinarySerDe.class, SequenceFileInputFormat.class,
+ return new TableDesc(SequenceFileInputFormat.class,
SequenceFileOutputFormat.class, Utilities.makeProperties("columns",
MetaStoreUtils.getColumnNamesFromFieldSchema(fieldSchemas),
"columns.types", MetaStoreUtils
.getColumnTypesFromFieldSchema(fieldSchemas),
- serdeConstants.ESCAPE_CHAR, "\\"));
+ serdeConstants.ESCAPE_CHAR, "\\",
+ serdeConstants.SERIALIZATION_LIB,LazyBinarySerDe.class.getName()));
}
/**
@@ -392,12 +391,13 @@ public final class PlanUtils {
*/
public static TableDesc getMapJoinValueTableDesc(
List<FieldSchema> fieldSchemas) {
- return new TableDesc(LazyBinarySerDe.class, SequenceFileInputFormat.class,
+ return new TableDesc(SequenceFileInputFormat.class,
SequenceFileOutputFormat.class, Utilities.makeProperties("columns",
MetaStoreUtils.getColumnNamesFromFieldSchema(fieldSchemas),
"columns.types", MetaStoreUtils
.getColumnTypesFromFieldSchema(fieldSchemas),
- serdeConstants.ESCAPE_CHAR, "\\"));
+ serdeConstants.ESCAPE_CHAR, "\\",
+ serdeConstants.SERIALIZATION_LIB,LazyBinarySerDe.class.getName()));
}
/**
@@ -405,26 +405,28 @@ public final class PlanUtils {
*/
public static TableDesc getIntermediateFileTableDesc(
List<FieldSchema> fieldSchemas) {
- return new TableDesc(LazyBinarySerDe.class, SequenceFileInputFormat.class,
+ return new TableDesc(SequenceFileInputFormat.class,
SequenceFileOutputFormat.class, Utilities.makeProperties(
serdeConstants.LIST_COLUMNS, MetaStoreUtils
.getColumnNamesFromFieldSchema(fieldSchemas),
serdeConstants.LIST_COLUMN_TYPES, MetaStoreUtils
.getColumnTypesFromFieldSchema(fieldSchemas),
- serdeConstants.ESCAPE_CHAR, "\\"));
+ serdeConstants.ESCAPE_CHAR, "\\",
+ serdeConstants.SERIALIZATION_LIB,LazyBinarySerDe.class.getName()));
}
/**
* Generate the table descriptor for intermediate files.
*/
public static TableDesc getReduceValueTableDesc(List<FieldSchema> fieldSchemas) {
- return new TableDesc(LazyBinarySerDe.class, SequenceFileInputFormat.class,
+ return new TableDesc(SequenceFileInputFormat.class,
SequenceFileOutputFormat.class, Utilities.makeProperties(
serdeConstants.LIST_COLUMNS, MetaStoreUtils
.getColumnNamesFromFieldSchema(fieldSchemas),
serdeConstants.LIST_COLUMN_TYPES, MetaStoreUtils
.getColumnTypesFromFieldSchema(fieldSchemas),
- serdeConstants.ESCAPE_CHAR, "\\"));
+ serdeConstants.ESCAPE_CHAR, "\\",
+ serdeConstants.SERIALIZATION_LIB,LazyBinarySerDe.class.getName()));
}
/**
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java?rev=1526201&r1=1526200&r2=1526201&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java Wed Sep 25 15:04:49 2013
@@ -24,9 +24,11 @@ import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Properties;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
import org.apache.hadoop.hive.ql.io.HivePassThroughOutputFormat;
+import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.mapred.InputFormat;
@@ -36,36 +38,35 @@ import org.apache.hadoop.mapred.InputFor
*/
public class TableDesc implements Serializable, Cloneable {
private static final long serialVersionUID = 1L;
- private Class<? extends Deserializer> deserializerClass;
private Class<? extends InputFormat> inputFileFormatClass;
private Class<? extends HiveOutputFormat> outputFileFormatClass;
private java.util.Properties properties;
- private String serdeClassName;
private Map<String, String> jobProperties;
public TableDesc() {
}
- public TableDesc(final Class<? extends Deserializer> serdeClass,
- final Class<? extends InputFormat> inputFileFormatClass,
- final Class<?> class1, final java.util.Properties properties) {
- deserializerClass = serdeClass;
- this.inputFileFormatClass = inputFileFormatClass;
+ /**
+ * @param inputFormatClass
+ * @param outputFormatClass
+ * @param properties must contain serde class name associate with this table.
+ */
+
+ public TableDesc(
+ final Class<? extends InputFormat> inputFormatClass,
+ final Class<?> outputFormatClass, final Properties properties) {
+ this.inputFileFormatClass = inputFormatClass;
outputFileFormatClass = HiveFileFormatUtils
- .getOutputFormatSubstitute(class1, false);
+ .getOutputFormatSubstitute(outputFormatClass, false);
this.properties = properties;
- serdeClassName = properties
- .getProperty(org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_LIB);
- ;
}
public Class<? extends Deserializer> getDeserializerClass() {
- return deserializerClass;
- }
-
- public void setDeserializerClass(
- final Class<? extends Deserializer> serdeClass) {
- deserializerClass = serdeClass;
+ try {
+ return (Class<? extends Deserializer>) Class.forName(getSerdeClassName());
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
}
public Class<? extends InputFormat> getInputFileFormatClass() {
@@ -76,7 +77,7 @@ public class TableDesc implements Serial
* Return a deserializer object corresponding to the tableDesc.
*/
public Deserializer getDeserializer() throws Exception {
- Deserializer de = deserializerClass.newInstance();
+ Deserializer de = getDeserializerClass().newInstance();
de.initialize(null, properties);
return de;
}
@@ -96,11 +97,11 @@ public class TableDesc implements Serial
}
@Explain(displayName = "properties", normalExplain = false)
- public java.util.Properties getProperties() {
+ public Properties getProperties() {
return properties;
}
- public void setProperties(final java.util.Properties properties) {
+ public void setProperties(final Properties properties) {
this.properties = properties;
}
@@ -118,21 +119,13 @@ public class TableDesc implements Serial
*/
@Explain(displayName = "serde")
public String getSerdeClassName() {
- return serdeClassName;
- }
-
- /**
- * @param serdeClassName
- * the serde Class Name to set
- */
- public void setSerdeClassName(String serdeClassName) {
- this.serdeClassName = serdeClassName;
+ return properties.getProperty(serdeConstants.SERIALIZATION_LIB);
}
@Explain(displayName = "name")
public String getTableName() {
return properties
- .getProperty(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_NAME);
+ .getProperty(hive_metastoreConstants.META_TABLE_NAME);
}
@Explain(displayName = "input format")
@@ -151,16 +144,12 @@ public class TableDesc implements Serial
}
public boolean isNonNative() {
- return (properties.getProperty(
- org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE)
- != null);
+ return (properties.getProperty(hive_metastoreConstants.META_TABLE_STORAGE) != null);
}
@Override
public Object clone() {
TableDesc ret = new TableDesc();
- ret.setSerdeClassName(serdeClassName);
- ret.setDeserializerClass(deserializerClass);
ret.setInputFileFormatClass(inputFileFormatClass);
ret.setOutputFileFormatClass(outputFileFormatClass);
Properties newProp = new Properties();
@@ -181,13 +170,11 @@ public class TableDesc implements Serial
public int hashCode() {
final int prime = 31;
int result = 1;
- result = prime * result + ((deserializerClass == null) ? 0 : deserializerClass.hashCode());
result = prime * result +
((inputFileFormatClass == null) ? 0 : inputFileFormatClass.hashCode());
result = prime * result +
((outputFileFormatClass == null) ? 0 : outputFileFormatClass.hashCode());
result = prime * result + ((properties == null) ? 0 : properties.hashCode());
- result = prime * result + ((serdeClassName == null) ? 0 : serdeClassName.hashCode());
result = prime * result + ((jobProperties == null) ? 0 : jobProperties.hashCode());
return result;
}
@@ -200,16 +187,12 @@ public class TableDesc implements Serial
TableDesc target = (TableDesc) o;
boolean ret = true;
- ret = ret && (deserializerClass == null ? target.deserializerClass == null :
- deserializerClass.equals(target.deserializerClass));
ret = ret && (inputFileFormatClass == null ? target.inputFileFormatClass == null :
inputFileFormatClass.equals(target.inputFileFormatClass));
ret = ret && (outputFileFormatClass == null ? target.outputFileFormatClass == null :
outputFileFormatClass.equals(target.outputFileFormatClass));
ret = ret && (properties == null ? target.properties == null :
properties.equals(target.properties));
- ret = ret && (serdeClassName == null ? target.serdeClassName == null :
- serdeClassName.equals(target.serdeClassName));
ret = ret && (jobProperties == null ? target.jobProperties == null :
jobProperties.equals(target.jobProperties));
return ret;
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCollectList.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCollectList.java?rev=1526201&r1=1526200&r2=1526201&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCollectList.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCollectList.java Wed Sep 25 15:04:49 2013
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.hadoop.hive.ql.udf.generic;
import org.apache.commons.logging.Log;
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMkCollectionEvaluator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMkCollectionEvaluator.java?rev=1526201&r1=1526200&r2=1526201&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMkCollectionEvaluator.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMkCollectionEvaluator.java Wed Sep 25 15:04:49 2013
@@ -1,5 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.hadoop.hive.ql.udf.generic;
+import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
@@ -12,14 +31,15 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector;
-import com.esotericsoftware.minlog.Log;
+public class GenericUDAFMkCollectionEvaluator extends GenericUDAFEvaluator
+ implements Serializable {
-public class GenericUDAFMkCollectionEvaluator extends GenericUDAFEvaluator {
+ private static final long serialVersionUID = 1l;
enum BufferType { SET, LIST }
// For PARTIAL1 and COMPLETE: ObjectInspectors for original data
- private PrimitiveObjectInspector inputOI;
+ private transient PrimitiveObjectInspector inputOI;
// For PARTIAL2 and FINAL: ObjectInspectors for partial aggregations (list
// of objs)
private transient StandardListObjectInspector loi;
@@ -29,8 +49,7 @@ public class GenericUDAFMkCollectionEval
private BufferType bufferType;
//needed by kyro
- public GenericUDAFMkCollectionEvaluator(){
-
+ public GenericUDAFMkCollectionEvaluator() {
}
public GenericUDAFMkCollectionEvaluator(BufferType bufferType){
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDate.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDate.java?rev=1526201&r1=1526200&r2=1526201&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDate.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDate.java Wed Sep 25 15:04:49 2013
@@ -23,6 +23,7 @@ import org.apache.hadoop.hive.ql.exec.UD
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter.DateConverter;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
@@ -51,11 +52,13 @@ public class GenericUDFToDate extends Ge
}
try {
argumentOI = (PrimitiveObjectInspector) arguments[0];
+ PrimitiveCategory pc = argumentOI.getPrimitiveCategory();
PrimitiveGrouping pg =
- PrimitiveObjectInspectorUtils.getPrimitiveGrouping(argumentOI.getPrimitiveCategory());
+ PrimitiveObjectInspectorUtils.getPrimitiveGrouping(pc);
switch (pg) {
case DATE_GROUP:
case STRING_GROUP:
+ case VOID_GROUP:
break;
default:
throw new UDFArgumentException(
Modified: hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/metastore/VerifyingObjectStore.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/metastore/VerifyingObjectStore.java?rev=1526201&r1=1526200&r2=1526201&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/metastore/VerifyingObjectStore.java (original)
+++ hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/metastore/VerifyingObjectStore.java Wed Sep 25 15:04:49 2013
@@ -26,6 +26,7 @@ import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.util.Collection;
import java.util.Iterator;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -37,6 +38,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.thrift.TException;
class VerifyingObjectStore extends ObjectStore {
private static final Log LOG = LogFactory.getLog(VerifyingObjectStore.class);
@@ -53,7 +55,7 @@ class VerifyingObjectStore extends Objec
dbName, tblName, filter, maxParts, true, false);
List<Partition> ormResults = getPartitionsByFilterInternal(
dbName, tblName, filter, maxParts, false, true);
- compareParts(sqlResults, ormResults);
+ verifyParts(sqlResults, ormResults);
return sqlResults;
}
@@ -64,21 +66,39 @@ class VerifyingObjectStore extends Objec
dbName, tblName, partNames, true, false);
List<Partition> ormResults = getPartitionsByNamesInternal(
dbName, tblName, partNames, false, true);
- compareParts(sqlResults, ormResults);
+ verifyParts(sqlResults, ormResults);
return sqlResults;
}
@Override
+ public boolean getPartitionsByExpr(String dbName, String tblName, byte[] expr,
+ String defaultPartitionName, short maxParts, Set<Partition> result) throws TException {
+ Set<Partition> ormParts = new LinkedHashSet<Partition>();
+ boolean sqlResult = getPartitionsByExprInternal(
+ dbName, tblName, expr, defaultPartitionName, maxParts, result, true, false);
+ boolean ormResult = getPartitionsByExprInternal(
+ dbName, tblName, expr, defaultPartitionName, maxParts, ormParts, false, true);
+ if (sqlResult != ormResult) {
+ String msg = "The unknown flag is different - SQL " + sqlResult + ", ORM " + ormResult;
+ LOG.error(msg);
+ throw new MetaException(msg);
+ }
+ verifyParts(result, ormParts);
+ return sqlResult;
+ }
+
+ @Override
public List<Partition> getPartitions(
String dbName, String tableName, int maxParts) throws MetaException {
List<Partition> sqlResults = getPartitionsInternal(dbName, tableName, maxParts, true, false);
List<Partition> ormResults = getPartitionsInternal(dbName, tableName, maxParts, false, true);
- compareParts(sqlResults, ormResults);
+ verifyParts(sqlResults, ormResults);
return sqlResults;
};
- private void compareParts(List<Partition> sqlResults, List<Partition> ormResults)
+ private void verifyParts(Collection<Partition> sqlResults, Collection<Partition> ormResults)
throws MetaException {
+ final int MAX_DIFFS = 5;
if (sqlResults.size() != ormResults.size()) {
String msg = "Lists are not the same size: SQL " + sqlResults.size()
+ ", ORM " + ormResults.size();
@@ -86,9 +106,12 @@ class VerifyingObjectStore extends Objec
throw new MetaException(msg);
}
+ Iterator<Partition> sqlIter = sqlResults.iterator(), ormIter = ormResults.iterator();
StringBuilder errorStr = new StringBuilder();
+ int errors = 0;
for (int partIx = 0; partIx < sqlResults.size(); ++partIx) {
- Partition p1 = sqlResults.get(partIx), p2 = ormResults.get(partIx);
+ assert sqlIter.hasNext() && ormIter.hasNext();
+ Partition p1 = sqlIter.next(), p2 = ormIter.next();
if (EqualsBuilder.reflectionEquals(p1, p2)) continue;
errorStr.append("Results are different at list index " + partIx + ": \n");
try {
@@ -102,6 +125,10 @@ class VerifyingObjectStore extends Objec
LOG.error(msg, t);
break;
}
+ if (++errors == MAX_DIFFS) {
+ errorStr.append("\n\nToo many diffs, giving up (lists might be sorted differently)");
+ break;
+ }
}
if (errorStr.length() > 0) {
LOG.error("Different results: \n" + errorStr.toString());
Modified: hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/serde2/TestSerDe.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/serde2/TestSerDe.java?rev=1526201&r1=1526200&r2=1526201&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/serde2/TestSerDe.java (original)
+++ hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/serde2/TestSerDe.java Wed Sep 25 15:04:49 2013
@@ -53,19 +53,6 @@ public class TestSerDe extends AbstractS
return "test_meta";
}
- static {
- StackTraceElement[] sTrace = new Exception().getStackTrace();
- String className = sTrace[0].getClassName();
- try {
- SerDeUtils.registerSerDe(shortName(), Class.forName(className));
- // For backward compatibility: this class replaces the following class.
- SerDeUtils.registerSerDe("org.apache.hadoop.hive.serde.TestSerDe", Class
- .forName(className));
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
public static final String DefaultSeparator = "\002";
private String separator;
Modified: hive/branches/vectorization/ql/src/test/queries/clientpositive/partition_date.q
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/queries/clientpositive/partition_date.q?rev=1526201&r1=1526200&r2=1526201&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/queries/clientpositive/partition_date.q (original)
+++ hive/branches/vectorization/ql/src/test/queries/clientpositive/partition_date.q Wed Sep 25 15:04:49 2013
@@ -12,7 +12,7 @@ insert overwrite table partition_date_1
select * from src limit 11;
select distinct dt from partition_date_1;
-select *, cast(dt as timestamp) from partition_date_1 where dt = '2000-01-01' and region = 2 order by key,value;
+select * from partition_date_1 where dt = '2000-01-01' and region = 2 order by key,value;
-- 15
select count(*) from partition_date_1 where dt = date '2000-01-01';
Modified: hive/branches/vectorization/ql/src/test/results/clientnegative/bucket_mapjoin_mismatch1.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientnegative/bucket_mapjoin_mismatch1.q.out?rev=1526201&r1=1526200&r2=1526201&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientnegative/bucket_mapjoin_mismatch1.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientnegative/bucket_mapjoin_mismatch1.q.out Wed Sep 25 15:04:49 2013
@@ -145,6 +145,7 @@ STAGE PLANS:
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -223,6 +224,7 @@ STAGE PLANS:
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Local Work:
Map Reduce Local Work
Modified: hive/branches/vectorization/ql/src/test/results/clientnegative/script_error.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientnegative/script_error.q.out?rev=1526201&r1=1526200&r2=1526201&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientnegative/script_error.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientnegative/script_error.q.out Wed Sep 25 15:04:49 2013
@@ -32,12 +32,14 @@ STAGE PLANS:
output info:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
File Output Operator
compressed: false
GlobalTableId: 0
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
Modified: hive/branches/vectorization/ql/src/test/results/clientnegative/sortmerge_mapjoin_mismatch_1.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientnegative/sortmerge_mapjoin_mismatch_1.q.out?rev=1526201&r1=1526200&r2=1526201&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientnegative/sortmerge_mapjoin_mismatch_1.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientnegative/sortmerge_mapjoin_mismatch_1.q.out Wed Sep 25 15:04:49 2013
@@ -122,6 +122,7 @@ STAGE PLANS:
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Local Work:
Map Reduce Local Work
Modified: hive/branches/vectorization/ql/src/test/results/clientnegative/udf_assert_true.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientnegative/udf_assert_true.q.out?rev=1526201&r1=1526200&r2=1526201&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientnegative/udf_assert_true.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientnegative/udf_assert_true.q.out Wed Sep 25 15:04:49 2013
@@ -37,6 +37,7 @@ STAGE PLANS:
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Select Operator
expressions:
expr: array(1,2)
@@ -58,6 +59,7 @@ STAGE PLANS:
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -108,6 +110,7 @@ STAGE PLANS:
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Select Operator
expressions:
expr: array(1,2)
@@ -129,6 +132,7 @@ STAGE PLANS:
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
Modified: hive/branches/vectorization/ql/src/test/results/clientnegative/udf_assert_true2.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientnegative/udf_assert_true2.q.out?rev=1526201&r1=1526200&r2=1526201&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientnegative/udf_assert_true2.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientnegative/udf_assert_true2.q.out Wed Sep 25 15:04:49 2013
@@ -32,6 +32,7 @@ STAGE PLANS:
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Select Operator
expressions:
expr: array(1,2)
@@ -53,6 +54,7 @@ STAGE PLANS:
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/alias_casted_column.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/alias_casted_column.q.out?rev=1526201&r1=1526200&r2=1526201&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/alias_casted_column.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/alias_casted_column.q.out Wed Sep 25 15:04:49 2013
@@ -29,6 +29,7 @@ STAGE PLANS:
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -66,6 +67,7 @@ STAGE PLANS:
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/allcolref_in_udf.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/allcolref_in_udf.q.out?rev=1526201&r1=1526200&r2=1526201&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/allcolref_in_udf.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/allcolref_in_udf.q.out Wed Sep 25 15:04:49 2013
@@ -36,6 +36,7 @@ STAGE PLANS:
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -158,6 +159,7 @@ STAGE PLANS:
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -227,6 +229,7 @@ STAGE PLANS:
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/alter_partition_coltype.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/alter_partition_coltype.q.out?rev=1526201&r1=1526200&r2=1526201&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/alter_partition_coltype.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/alter_partition_coltype.q.out Wed Sep 25 15:04:49 2013
@@ -205,6 +205,8 @@ STAGE PLANS:
escape.delim \
hive.serialization.extend.nesting.levels true
serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
TotalFiles: 1
GatherStats: false
MultiFileSpray: false
@@ -392,6 +394,8 @@ STAGE PLANS:
escape.delim \
hive.serialization.extend.nesting.levels true
serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
TotalFiles: 1
GatherStats: false
MultiFileSpray: false
@@ -642,6 +646,8 @@ STAGE PLANS:
escape.delim \
hive.serialization.extend.nesting.levels true
serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
TotalFiles: 1
GatherStats: false
MultiFileSpray: false
@@ -799,6 +805,8 @@ STAGE PLANS:
escape.delim \
hive.serialization.extend.nesting.levels true
serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
TotalFiles: 1
GatherStats: false
MultiFileSpray: false
@@ -1208,6 +1216,8 @@ STAGE PLANS:
escape.delim \
hive.serialization.extend.nesting.levels true
serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
TotalFiles: 1
GatherStats: false
MultiFileSpray: false
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/ambiguous_col.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/ambiguous_col.q.out?rev=1526201&r1=1526200&r2=1526201&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/ambiguous_col.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/ambiguous_col.q.out Wed Sep 25 15:04:49 2013
@@ -78,6 +78,7 @@ STAGE PLANS:
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -158,6 +159,7 @@ STAGE PLANS:
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -238,6 +240,7 @@ STAGE PLANS:
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/auto_join0.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/auto_join0.q.out?rev=1526201&r1=1526200&r2=1526201&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/auto_join0.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/auto_join0.q.out Wed Sep 25 15:04:49 2013
@@ -149,6 +149,7 @@ STAGE PLANS:
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
Stage: Stage-3
Map Reduce
@@ -179,6 +180,7 @@ STAGE PLANS:
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/auto_join10.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/auto_join10.q.out?rev=1526201&r1=1526200&r2=1526201&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/auto_join10.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/auto_join10.q.out Wed Sep 25 15:04:49 2013
@@ -112,6 +112,7 @@ STAGE PLANS:
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/auto_join11.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/auto_join11.q.out?rev=1526201&r1=1526200&r2=1526201&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/auto_join11.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/auto_join11.q.out Wed Sep 25 15:04:49 2013
@@ -120,6 +120,7 @@ STAGE PLANS:
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator