You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2011/03/22 17:33:11 UTC
svn commit: r1084240 - in /hive/trunk:
metastore/src/java/org/apache/hadoop/hive/metastore/
metastore/src/java/org/apache/hadoop/hive/metastore/parser/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/metadata/ ql/src/j...
Author: namit
Date: Tue Mar 22 16:33:10 2011
New Revision: 1084240
URL: http://svn.apache.org/viewvc?rev=1084240&view=rev
Log:
HIVE-2049 Push down partition pruning to JDO filtering for a subset
of partition predicates (Ning Zhang via namit)
Added:
hive/trunk/ql/src/test/queries/clientpositive/ppr_pushdown.q
hive/trunk/ql/src/test/queries/clientpositive/ppr_pushdown2.q
hive/trunk/ql/src/test/results/clientpositive/ppr_pushdown.q.out
hive/trunk/ql/src/test/results/clientpositive/ppr_pushdown2.q.out
Modified:
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java?rev=1084240&r1=1084239&r2=1084240&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java Tue Mar 22 16:33:10 2011
@@ -269,7 +269,7 @@ public interface IMetaStoreClient {
public Partition getPartition(String dbName, String tblName,
String name) throws MetaException, UnknownTableException, NoSuchObjectException, TException;
-
+
/**
* @param dbName
* @param tableName
@@ -285,7 +285,7 @@ public interface IMetaStoreClient {
public Partition getPartitionWithAuthInfo(String dbName, String tableName,
List<String> pvals, String userName, List<String> groupNames)
throws MetaException, UnknownTableException, NoSuchObjectException, TException;
-
+
/**
* @param tbl_name
* @param db_name
@@ -308,13 +308,31 @@ public interface IMetaStoreClient {
List<String> part_vals, short max_parts) throws MetaException, TException;
/**
+ * Get list of partitions matching specified filter
+ * @param db_name the database name
+ * @param tbl_name the table name
+ * @param filter the filter string,
+ * for example "part1 = \"p1_abc\" and part2 <= "\p2_test\"". Filtering can
+ * be done only on string partition keys.
+ * @param max_parts the maximum number of partitions to return,
+ * all partitions are returned if -1 is passed
+ * @return list of partitions
+ * @throws MetaException
+ * @throws NoSuchObjectException
+ * @throws TException
+ */
+ public List<Partition> listPartitionsByFilter(String db_name, String tbl_name,
+ String filter, short max_parts) throws MetaException,
+ NoSuchObjectException, TException;
+
+ /**
* @param dbName
* @param tableName
* @param s
* @param userName
* @param groupNames
* @return
- * @throws NoSuchObjectException
+ * @throws NoSuchObjectException
*/
public List<Partition> listPartitionsWithAuthInfo(String dbName,
String tableName, short s, String userName, List<String> groupNames)
@@ -328,7 +346,7 @@ public interface IMetaStoreClient {
* @param userName
* @param groupNames
* @return
- * @throws NoSuchObjectException
+ * @throws NoSuchObjectException
*/
public List<Partition> listPartitionsWithAuthInfo(String dbName,
String tableName, List<String> partialPvals, short s, String userName,
@@ -531,7 +549,7 @@ public interface IMetaStoreClient {
public boolean dropIndex(String db_name, String tbl_name,
String name, boolean deleteData) throws NoSuchObjectException,
MetaException, TException;
-
+
/**
* @param Role
* role object
@@ -545,8 +563,8 @@ public interface IMetaStoreClient {
/**
* @param role_name
* role name
- * @param db_name
- *
+ * @param db_name
+ *
* @return
* @throws MetaException
* @throws TException
@@ -556,13 +574,13 @@ public interface IMetaStoreClient {
/**
* list all role names
* @return
- * @throws TException
- * @throws MetaException
+ * @throws TException
+ * @throws MetaException
*/
public List<String> listRoleNames() throws MetaException, TException;
/**
- *
+ *
* @param role_name
* @param user_name
* @param principalType
@@ -584,7 +602,7 @@ public interface IMetaStoreClient {
* user name
* @param principalType
* @param db_name
- *
+ *
* @return
* @throws MetaException
* @throws TException
@@ -593,7 +611,7 @@ public interface IMetaStoreClient {
PrincipalType principalType) throws MetaException, TException;
/**
- *
+ *
* @param principalName
* @param principalType
* @return
@@ -614,7 +632,7 @@ public interface IMetaStoreClient {
public PrincipalPrivilegeSet get_privilege_set(HiveObjectRef hiveObject,
String user_name, List<String> group_names) throws MetaException,
TException;
-
+
/**
* @param principal_name
* @param principal_type
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java?rev=1084240&r1=1084239&r2=1084240&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java Tue Mar 22 16:33:10 2011
@@ -1221,6 +1221,7 @@ public class ObjectStore implements RawS
Query query = pm.newQuery(MPartition.class,
"table.tableName == t1 && table.database.name == t2");
query.declareParameters("java.lang.String t1, java.lang.String t2");
+ query.setOrdering("partitionName ascending");
mparts = (List<MPartition>) query.execute(tableName, dbName);
LOG.debug("Done executing query for listMPartitions");
pm.retrieveAll(mparts);
@@ -1240,6 +1241,7 @@ public class ObjectStore implements RawS
openTransaction();
List<Partition> parts = convertToParts(listMPartitionsByFilter(dbName,
tblName, filter, maxParts));
+ LOG.info("# parts after pruning = " + parts.size());
commitTransaction();
return parts;
}
@@ -1269,6 +1271,7 @@ public class ObjectStore implements RawS
}
String jdoFilter = parser.tree.generateJDOFilter(table, params);
+ LOG.debug("jdoFilter = " + jdoFilter);
if( jdoFilter.trim().length() > 0 ) {
queryBuilder.append(" && ( ");
@@ -1276,7 +1279,6 @@ public class ObjectStore implements RawS
queryBuilder.append(" )");
}
}
-
return queryBuilder.toString();
}
@@ -1347,7 +1349,7 @@ public class ObjectStore implements RawS
List<String> partNames = new ArrayList<String>();
try {
openTransaction();
- LOG.debug("Executing listMPartitionsByFilter");
+ LOG.debug("Executing listMPartitionNamesByFilter");
dbName = dbName.toLowerCase();
tableName = tableName.toLowerCase();
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java?rev=1084240&r1=1084239&r2=1084240&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java Tue Mar 22 16:33:10 2011
@@ -23,11 +23,11 @@ import java.util.Stack;
import org.antlr.runtime.ANTLRStringStream;
import org.antlr.runtime.CharStream;
+import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.serde.Constants;
-import org.apache.hadoop.hive.common.FileUtils;
/**
* The Class representing the filter as a binary tree. The tree has TreeNode's
@@ -239,19 +239,30 @@ public class ExpressionTree {
// makePartName with one key will return a substring of the name made
// with both all the keys.
String escapedNameFragment = Warehouse.makePartName(partKeyToVal, false);
-
+ StringBuilder fltr = new StringBuilder();
if (keyCount == 1) {
// Case where this is no other partition columns
params.put(paramName, escapedNameFragment);
+ fltr.append("partitionName == ").append(paramName);
} else if (keyPos + 1 == keyCount) {
// Case where the partition column is at the end of the name. There will
// be a leading '/' but no trailing '/'
- params.put(paramName, ".*/" + escapedNameFragment);
+ params.put(paramName, "/" + escapedNameFragment);
+ fltr.append("partitionName.endsWith(").append(paramName).append(')');
+ } else if (keyPos == 0) {
+ // Case where the parttion column is at the beginning of the name. There will
+ // be a trailing '/' but no leading '/'
+ params.put(paramName, escapedNameFragment + "/");
+ fltr.append("partitionName.startsWith(").append(paramName).append(')');
} else {
- params.put(paramName, ".*" + escapedNameFragment + "/.*");
+ // Case where the partition column is in the middle of the name. There will
+ // be a leading '/' and an trailing '/'
+ params.put(paramName, "/" + escapedNameFragment + "/");
+ fltr.append("partitionName.indexOf(").append(paramName).append(") >= 0");
}
- return "partitionName.matches(" + paramName + ")";
+ return fltr.toString();
}
+
/**
* The root node for the tree.
*/
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1084240&r1=1084239&r2=1084240&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Tue Mar 22 16:33:10 2011
@@ -99,7 +99,10 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.ql.parse.ErrorMsg;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx;
+import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.plan.FileSinkDesc;
import org.apache.hadoop.hive.ql.plan.GroupByDesc;
import org.apache.hadoop.hive.ql.plan.MapredLocalWork;
@@ -110,10 +113,15 @@ import org.apache.hadoop.hive.ql.plan.Ta
import org.apache.hadoop.hive.ql.plan.PlanUtils.ExpressionTypes;
import org.apache.hadoop.hive.ql.stats.StatsFactory;
import org.apache.hadoop.hive.ql.stats.StatsPublisher;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr;
import org.apache.hadoop.hive.serde.Constants;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.Serializer;
import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.SequenceFile;
@@ -1894,6 +1902,66 @@ public final class Utilities {
}
}
+ /**
+ * Check if a function can be pushed down to JDO.
+ * Now only {=, AND, OR} are supported.
+ * @param func a generic function.
+ * @return true if this function can be pushed down to JDO filter.
+ */
+ private static boolean supportedJDOFuncs(GenericUDF func) {
+ return func instanceof GenericUDFOPEqual ||
+ func instanceof GenericUDFOPAnd ||
+ func instanceof GenericUDFOPOr;
+ }
+
+ /**
+ * Check if the partition pruning expression can be pushed down to JDO filtering.
+ * The partition expression contains only partition columns.
+ * The criteria that an expression can be pushed down are that:
+ * 1) the expression only contains function specified in supportedJDOFuncs().
+ * Now only {=, AND, OR} can be pushed down.
+ * 2) the partition column type and the constant type have to be String. This is
+ * restriction by the current JDO filtering implementation.
+ * @param tab The table that contains the partition columns.
+ * @param expr the partition pruning expression
+ * @return true if the partition pruning expression can be pushed down to JDO filtering.
+ */
+ public static boolean checkJDOPushDown(Table tab, ExprNodeDesc expr) {
+ if (expr instanceof ExprNodeConstantDesc) {
+ // JDO filter now only support String typed literal -- see Filter.g and ExpressionTree.java
+ Object value = ((ExprNodeConstantDesc)expr).getValue();
+ return (value instanceof String);
+ } else if (expr instanceof ExprNodeColumnDesc) {
+ // JDO filter now only support String typed literal -- see Filter.g and ExpressionTree.java
+ TypeInfo type = expr.getTypeInfo();
+ if (type.getTypeName().equals(Constants.STRING_TYPE_NAME)) {
+ String colName = ((ExprNodeColumnDesc)expr).getColumn();
+ for (FieldSchema fs: tab.getPartCols()) {
+ if (fs.getName().equals(colName)) {
+ return fs.getType().equals(Constants.STRING_TYPE_NAME);
+ }
+ }
+ assert(false); // cannot find the partition column!
+ } else {
+ return false;
+ }
+ } else if (expr instanceof ExprNodeGenericFuncDesc) {
+ ExprNodeGenericFuncDesc funcDesc = (ExprNodeGenericFuncDesc) expr;
+ GenericUDF func = funcDesc.getGenericUDF();
+ if (!supportedJDOFuncs(func)) {
+ return false;
+ }
+ List<ExprNodeDesc> children = funcDesc.getChildExprs();
+ for (ExprNodeDesc child: children) {
+ if (!checkJDOPushDown(tab, child)) {
+ return false;
+ }
+ }
+ return true;
+ }
+ return false;
+ }
+
private static ThreadLocal<Map<String, Long>> perfKeyMaps = new ThreadLocal<Map<String, Long>>();
/**
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java?rev=1084240&r1=1084239&r2=1084240&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java Tue Mar 22 16:33:10 2011
@@ -1045,7 +1045,7 @@ public class Hive {
* if true - replace files in the partition, otherwise add files to
* the partition
* @param holdDDLTime if true, force [re]create the partition
- * @param inheritTableSpecs if true, on [re]creating the partition, take the
+ * @param inheritTableSpecs if true, on [re]creating the partition, take the
* location/inputformat/outputformat/serde details from table spec
* @param tmpDirPath
* The temporary directory.
@@ -1071,7 +1071,7 @@ public class Hive {
Path newPartPath = null;
-
+
if (inheritTableSpecs) {
Path partPath = new Path(tbl.getDataLocation().getPath(),
Warehouse.makePartPath(partSpec));
@@ -1080,7 +1080,7 @@ public class Hive {
} else {
newPartPath = oldPartPath;
}
-
+
if (replace) {
Hive.replaceFiles(loadPath, newPartPath, oldPartPath, getConf());
} else {
@@ -1208,7 +1208,7 @@ public class Hive {
*/
public Partition createPartition(Table tbl, Map<String, String> partSpec)
throws HiveException {
- return createPartition(tbl, partSpec, null, null, null, null, -1,
+ return createPartition(tbl, partSpec, null, null, null, null, -1,
null, null, null, null, null);
}
@@ -1231,7 +1231,7 @@ public class Hive {
* @param serdeParams the serde parameters
* @param bucketCols the bucketing columns
* @param sortCols sort columns and order
- *
+ *
* @return created partition object
* @throws HiveException
* if table doesn't exist or partition already exists
@@ -1310,7 +1310,7 @@ public class Hive {
* if this is true and partition doesn't exist then a partition is
* created
* @param partPath the path where the partition data is located
- * @param inheritTableSpecs whether to copy over the table specs for if/of/serde
+ * @param inheritTableSpecs whether to copy over the table specs for if/of/serde
* @return result partition object or null if there is no partition
* @throws HiveException
*/
@@ -1544,6 +1544,36 @@ public class Hive {
return partitions;
}
+
+ /**
+ * Get a list of Partitions by filter.
+ * @param tbl The table containing the partitions.
+ * @param filter A string represent partition predicates.
+ * @return a list of partitions satisfying the partition predicates.
+ * @throws HiveException
+ * @throws MetaException
+ * @throws NoSuchObjectException
+ * @throws TException
+ */
+ public List<Partition> getPartitionsByFilter(Table tbl, String filter)
+ throws HiveException, MetaException, NoSuchObjectException, TException {
+
+ if (!tbl.isPartitioned()) {
+ throw new HiveException("Partition spec should only be supplied for a " +
+ "partitioned table");
+ }
+
+ List<org.apache.hadoop.hive.metastore.api.Partition> tParts = getMSC().listPartitionsByFilter(
+ tbl.getDbName(), tbl.getTableName(), filter, (short)-1);
+ List<Partition> results = new ArrayList<Partition>(tParts.size());
+
+ for (org.apache.hadoop.hive.metastore.api.Partition tPart: tParts) {
+ Partition part = new Partition(tbl, tPart);
+ results.add(part);
+ }
+ return results;
+ }
+
/**
* Get the name of the current database
* @return
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java?rev=1084240&r1=1084239&r2=1084240&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java Tue Mar 22 16:33:10 2011
@@ -23,12 +23,16 @@ import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
+import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher;
import org.apache.hadoop.hive.ql.lib.Dispatcher;
@@ -47,9 +51,14 @@ import org.apache.hadoop.hive.ql.parse.P
import org.apache.hadoop.hive.ql.parse.PrunedPartitionList;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.thrift.TException;
/**
* The transformation step that does partition pruning.
@@ -177,59 +186,46 @@ public class PartitionPruner implements
Object[] rowWithPart = new Object[2];
if (tab.isPartitioned()) {
- LOG.debug("tabname = " + tab.getTableName() + " is partitioned");
-
- for (String partName : Hive.get().getPartitionNames(tab.getDbName(),
- tab.getTableName(), (short) -1)) {
- // If the "strict" mode is on, we have to provide partition pruner for
- // each table.
- if ("strict".equalsIgnoreCase(HiveConf.getVar(conf,
- HiveConf.ConfVars.HIVEMAPREDMODE))) {
- if (!hasColumnExpr(prunerExpr)) {
- throw new SemanticException(ErrorMsg.NO_PARTITION_PREDICATE
- .getMsg("for Alias \"" + alias + "\" Table \""
- + tab.getTableName() + "\""));
- }
+ // If the "strict" mode is on, we have to provide partition pruner for
+ // each table.
+ if ("strict".equalsIgnoreCase(HiveConf.getVar(conf,
+ HiveConf.ConfVars.HIVEMAPREDMODE))) {
+ if (!hasColumnExpr(prunerExpr)) {
+ throw new SemanticException(ErrorMsg.NO_PARTITION_PREDICATE
+ .getMsg("for Alias \"" + alias + "\" Table \""
+ + tab.getTableName() + "\""));
}
+ }
- // Set all the variables here
- LinkedHashMap<String, String> partSpec = Warehouse
- .makeSpecFromName(partName);
-
- LOG.trace("about to process partition " + partSpec + " for pruning ");
- // evaluate the expression tree
- if (prunerExpr != null) {
- Boolean r = (Boolean) PartExprEvalUtils.evalExprWithPart(prunerExpr, partSpec,
- rowObjectInspector);
-
- LOG.trace("prune result for partition " + partSpec + ": " + r);
- if (Boolean.FALSE.equals(r)) {
- if (denied_parts.isEmpty()) {
- Partition part = Hive.get().getPartition(tab, partSpec,
- Boolean.FALSE);
- denied_parts.add(part);
- }
- LOG.trace("pruned partition: " + partSpec);
+ if (prunerExpr == null) {
+ // add all partitions corresponding to the table
+ true_parts.addAll(Hive.get().getPartitions(tab));
+ } else {
+ // remove non-partition columns
+ ExprNodeDesc compactExpr = prunerExpr.clone();
+ compactExpr = compactExpr(compactExpr);
+ LOG.debug("Filter w/ compacting: " +
+ ((compactExpr != null) ? compactExpr.getExprString(): "null") +
+ "; filter w/o compacting: " +
+ ((prunerExpr != null) ? prunerExpr.getExprString(): "null"));
+ if (compactExpr == null) {
+ true_parts.addAll(Hive.get().getPartitions(tab));
+ } else if (Utilities.checkJDOPushDown(tab, compactExpr)) {
+ String filter = compactExpr.getExprString();
+ String oldFilter = prunerExpr.getExprString();
+
+ if (filter.equals(oldFilter)) {
+ // pruneExpr contains only partition columns
+ pruneByPushDown(tab, true_parts, filter);
} else {
- Partition part = Hive.get().getPartition(tab, partSpec,
- Boolean.FALSE);
- String state = "retained";
- if (Boolean.TRUE.equals(r)) {
- true_parts.add(part);
- } else {
- unkn_parts.add(part);
- state = "unknown";
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug(state + " partition: " + partSpec);
- }
+ // pruneExpr contains non-partition columns
+ pruneByPushDown(tab, unkn_parts, filter);
}
} else {
- // is there is no parition pruning, all of them are needed
- true_parts.add(Hive.get()
- .getPartition(tab, partSpec, Boolean.FALSE));
+ pruneBySequentialScan(tab, true_parts, unkn_parts, denied_parts, prunerExpr, rowObjectInspector);
}
}
+ LOG.debug("tabname = " + tab.getTableName() + " is partitioned");
} else {
true_parts.addAll(Hive.get().getPartitions(tab));
}
@@ -246,6 +242,120 @@ public class PartitionPruner implements
}
/**
+ * Taking a partition pruning expression, remove the null operands.
+ * @param expr original partition pruning expression.
+ * @return partition pruning expression that only contains partition columns.
+ */
+ static private ExprNodeDesc compactExpr(ExprNodeDesc expr) {
+ if (expr instanceof ExprNodeConstantDesc) {
+ if (((ExprNodeConstantDesc)expr).getValue() == null) {
+ return null;
+ } else {
+ return expr;
+ }
+ } else if (expr instanceof ExprNodeGenericFuncDesc) {
+ GenericUDF udf = ((ExprNodeGenericFuncDesc)expr).getGenericUDF();
+ if (udf instanceof GenericUDFOPAnd ||
+ udf instanceof GenericUDFOPOr) {
+ List<ExprNodeDesc> children = ((ExprNodeGenericFuncDesc)expr).getChildren();
+ ExprNodeDesc left = children.get(0);
+ children.set(0, compactExpr(left));
+ ExprNodeDesc right = children.get(1);
+ children.set(1, compactExpr(right));
+ if (children.get(0) == null && children.get(1) == null) {
+ return null;
+ } else if (children.get(0) == null) {
+ return children.get(1);
+ } else if (children.get(1) == null) {
+ return children.get(0);
+ }
+ }
+ return expr;
+ }
+ return expr;
+ }
+
+ /**
+ * Pruning partition using JDO filtering.
+ * @param tab the table containing the partitions.
+ * @param true_parts the resulting partitions.
+ * @param filter the SQL predicate that involves only partition columns
+ * @throws HiveException
+ * @throws MetaException
+ * @throws NoSuchObjectException
+ * @throws TException
+ */
+ static private void pruneByPushDown(Table tab, Set<Partition> true_parts, String filter)
+ throws HiveException, MetaException, NoSuchObjectException, TException {
+ Hive db = Hive.get();
+ List<Partition> parts = db.getPartitionsByFilter(tab, filter);
+ true_parts.addAll(parts);
+ return;
+ }
+
+ /**
+ * Pruning partition by getting the partition names first and pruning using Hive expression
+ * evaluator.
+ * @param tab the table containing the partitions.
+ * @param true_parts the resulting partitions if the partition pruning expression only contains
+ * partition columns.
+ * @param unkn_parts the resulting partitions if the partition pruning expression that only contains
+ * non-partition columns.
+ * @param denied_parts pruned out partitions.
+ * @param prunerExpr the SQL predicate that involves partition columns.
+ * @param rowObjectInspector object inspector used by the evaluator
+ * @throws Exception
+ */
+ static private void pruneBySequentialScan(Table tab, Set<Partition> true_parts, Set<Partition> unkn_parts,
+ Set<Partition> denied_parts, ExprNodeDesc prunerExpr, StructObjectInspector rowObjectInspector)
+ throws Exception {
+
+ for (String partName : Hive.get().getPartitionNames(tab.getDbName(),
+ tab.getTableName(), (short) -1)) {
+
+ // Set all the variables here
+ LinkedHashMap<String, String> partSpec = Warehouse
+ .makeSpecFromName(partName);
+
+ LOG.trace("about to process partition " + partSpec + " for pruning ");
+ // evaluate the expression tree
+ if (prunerExpr != null) {
+
+ Boolean r = (Boolean) PartExprEvalUtils.evalExprWithPart(prunerExpr, partSpec,
+ rowObjectInspector);
+
+ if (Boolean.FALSE.equals(r)) {
+ if (denied_parts.isEmpty()) {
+ Partition part = Hive.get().getPartition(tab, partSpec,
+ Boolean.FALSE);
+ denied_parts.add(part);
+ }
+ LOG.trace("pruned partition: " + partSpec);
+ } else {
+ Partition part = Hive.get().getPartition(tab, partSpec,
+ Boolean.FALSE);
+ String state = "retained";
+ if (Boolean.TRUE.equals(r)) {
+ true_parts.add(part);
+ } else {
+ // r == null means prunerExpr contains null subexpression,
+ // which was converted from non-partition columns
+ assert (r == null);
+ unkn_parts.add(part);
+ state = "unknown";
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(state + " partition: " + partSpec);
+ }
+ }
+ } else {
+ // is there is no parition pruning, all of them are needed
+ true_parts.add(Hive.get()
+ .getPartition(tab, partSpec, Boolean.FALSE));
+ }
+ }
+ }
+ /**
* Whether the expression contains a column node or not.
*/
public static boolean hasColumnExpr(ExprNodeDesc desc) {
Added: hive/trunk/ql/src/test/queries/clientpositive/ppr_pushdown.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/ppr_pushdown.q?rev=1084240&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/ppr_pushdown.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/ppr_pushdown.q Tue Mar 22 16:33:10 2011
@@ -0,0 +1,41 @@
+create table ppr_test (key string) partitioned by (ds string);
+
+alter table ppr_test add partition (ds = '1234');
+alter table ppr_test add partition (ds = '1224');
+alter table ppr_test add partition (ds = '1214');
+alter table ppr_test add partition (ds = '12+4');
+alter table ppr_test add partition (ds = '12.4');
+alter table ppr_test add partition (ds = '12:4');
+alter table ppr_test add partition (ds = '12%4');
+alter table ppr_test add partition (ds = '12*4');
+
+insert overwrite table ppr_test partition(ds = '1234') select * from (select '1234' from src limit 1 union all select 'abcd' from src limit 1) s;
+insert overwrite table ppr_test partition(ds = '1224') select * from (select '1224' from src limit 1 union all select 'abcd' from src limit 1) s;
+insert overwrite table ppr_test partition(ds = '1214') select * from (select '1214' from src limit 1 union all select 'abcd' from src limit 1) s;
+insert overwrite table ppr_test partition(ds = '12+4') select * from (select '12+4' from src limit 1 union all select 'abcd' from src limit 1) s;
+insert overwrite table ppr_test partition(ds = '12.4') select * from (select '12.4' from src limit 1 union all select 'abcd' from src limit 1) s;
+insert overwrite table ppr_test partition(ds = '12:4') select * from (select '12:4' from src limit 1 union all select 'abcd' from src limit 1) s;
+insert overwrite table ppr_test partition(ds = '12%4') select * from (select '12%4' from src limit 1 union all select 'abcd' from src limit 1) s;
+insert overwrite table ppr_test partition(ds = '12*4') select * from (select '12*4' from src limit 1 union all select 'abcd' from src limit 1) s;
+
+
+select * from ppr_test where ds = '1234';
+select * from ppr_test where ds = '1224';
+select * from ppr_test where ds = '1214';
+select * from ppr_test where ds = '12.4';
+select * from ppr_test where ds = '12+4';
+select * from ppr_test where ds = '12:4';
+select * from ppr_test where ds = '12%4';
+select * from ppr_test where ds = '12*4';
+select * from ppr_test where ds = '12.*4';
+
+select * from ppr_test where ds = '1234' and key = '1234';
+select * from ppr_test where ds = '1224' and key = '1224';
+select * from ppr_test where ds = '1214' and key = '1214';
+select * from ppr_test where ds = '12.4' and key = '12.4';
+select * from ppr_test where ds = '12+4' and key = '12+4';
+select * from ppr_test where ds = '12:4' and key = '12:4';
+select * from ppr_test where ds = '12%4' and key = '12%4';
+select * from ppr_test where ds = '12*4' and key = '12*4';
+
+
Added: hive/trunk/ql/src/test/queries/clientpositive/ppr_pushdown2.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/ppr_pushdown2.q?rev=1084240&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/ppr_pushdown2.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/ppr_pushdown2.q Tue Mar 22 16:33:10 2011
@@ -0,0 +1,29 @@
+create table ppr_test (key string) partitioned by (ds string);
+
+insert overwrite table ppr_test partition(ds='2') select '2' from src limit 1;
+insert overwrite table ppr_test partition(ds='22') select '22' from src limit 1;
+
+select * from ppr_test where ds = '2';
+select * from ppr_test where ds = '22';
+
+
+create table ppr_test2 (key string) partitioned by (ds string, s string);
+insert overwrite table ppr_test2 partition(ds='1', s='2') select '1' from src limit 1;
+insert overwrite table ppr_test2 partition(ds='2', s='1') select '2' from src limit 1;
+
+select * from ppr_test2 where s = '1';
+select * from ppr_test2 where ds = '1';
+
+
+create table ppr_test3 (key string) partitioned by (col string, ol string, l string);
+insert overwrite table ppr_test3 partition(col='1', ol='2', l = '3') select '1' from src limit 1;
+insert overwrite table ppr_test3 partition(col='1', ol='1', l = '2') select '2' from src limit 1;
+insert overwrite table ppr_test3 partition(col='1', ol='2', l = '1') select '3' from src limit 1;
+
+select * from ppr_test3 where l = '1';
+select * from ppr_test3 where l = '2';
+select * from ppr_test3 where ol = '1';
+select * from ppr_test3 where ol = '2';
+select * from ppr_test3 where col = '1';
+select * from ppr_test3 where ol = '2' and l = '1';
+select * from ppr_test3 where col='1' and ol = '2' and l = '1';
Added: hive/trunk/ql/src/test/results/clientpositive/ppr_pushdown.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/ppr_pushdown.q.out?rev=1084240&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/ppr_pushdown.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/ppr_pushdown.q.out Tue Mar 22 16:33:10 2011
@@ -0,0 +1,455 @@
+PREHOOK: query: create table ppr_test (key string) partitioned by (ds string)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table ppr_test (key string) partitioned by (ds string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@ppr_test
+PREHOOK: query: alter table ppr_test add partition (ds = '1234')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: default@ppr_test
+POSTHOOK: query: alter table ppr_test add partition (ds = '1234')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: default@ppr_test
+POSTHOOK: Output: default@ppr_test@ds=1234
+PREHOOK: query: alter table ppr_test add partition (ds = '1224')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: default@ppr_test
+POSTHOOK: query: alter table ppr_test add partition (ds = '1224')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: default@ppr_test
+POSTHOOK: Output: default@ppr_test@ds=1224
+PREHOOK: query: alter table ppr_test add partition (ds = '1214')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: default@ppr_test
+POSTHOOK: query: alter table ppr_test add partition (ds = '1214')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: default@ppr_test
+POSTHOOK: Output: default@ppr_test@ds=1214
+PREHOOK: query: alter table ppr_test add partition (ds = '12+4')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: default@ppr_test
+POSTHOOK: query: alter table ppr_test add partition (ds = '12+4')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: default@ppr_test
+POSTHOOK: Output: default@ppr_test@ds=12+4
+PREHOOK: query: alter table ppr_test add partition (ds = '12.4')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: default@ppr_test
+POSTHOOK: query: alter table ppr_test add partition (ds = '12.4')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: default@ppr_test
+POSTHOOK: Output: default@ppr_test@ds=12.4
+PREHOOK: query: alter table ppr_test add partition (ds = '12:4')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: default@ppr_test
+POSTHOOK: query: alter table ppr_test add partition (ds = '12:4')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: default@ppr_test
+POSTHOOK: Output: default@ppr_test@ds=12%3A4
+PREHOOK: query: alter table ppr_test add partition (ds = '12%4')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: default@ppr_test
+POSTHOOK: query: alter table ppr_test add partition (ds = '12%4')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: default@ppr_test
+POSTHOOK: Output: default@ppr_test@ds=12%254
+PREHOOK: query: alter table ppr_test add partition (ds = '12*4')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: default@ppr_test
+POSTHOOK: query: alter table ppr_test add partition (ds = '12*4')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: default@ppr_test
+POSTHOOK: Output: default@ppr_test@ds=12%2A4
+PREHOOK: query: insert overwrite table ppr_test partition(ds = '1234') select * from (select '1234' from src limit 1 union all select 'abcd' from src limit 1) s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@ppr_test@ds=1234
+POSTHOOK: query: insert overwrite table ppr_test partition(ds = '1234') select * from (select '1234' from src limit 1 union all select 'abcd' from src limit 1) s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@ppr_test@ds=1234
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION []
+PREHOOK: query: insert overwrite table ppr_test partition(ds = '1224') select * from (select '1224' from src limit 1 union all select 'abcd' from src limit 1) s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@ppr_test@ds=1224
+POSTHOOK: query: insert overwrite table ppr_test partition(ds = '1224') select * from (select '1224' from src limit 1 union all select 'abcd' from src limit 1) s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@ppr_test@ds=1224
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION []
+PREHOOK: query: insert overwrite table ppr_test partition(ds = '1214') select * from (select '1214' from src limit 1 union all select 'abcd' from src limit 1) s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@ppr_test@ds=1214
+POSTHOOK: query: insert overwrite table ppr_test partition(ds = '1214') select * from (select '1214' from src limit 1 union all select 'abcd' from src limit 1) s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@ppr_test@ds=1214
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION []
+PREHOOK: query: insert overwrite table ppr_test partition(ds = '12+4') select * from (select '12+4' from src limit 1 union all select 'abcd' from src limit 1) s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@ppr_test@ds=12+4
+POSTHOOK: query: insert overwrite table ppr_test partition(ds = '12+4') select * from (select '12+4' from src limit 1 union all select 'abcd' from src limit 1) s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@ppr_test@ds=12+4
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION []
+PREHOOK: query: insert overwrite table ppr_test partition(ds = '12.4') select * from (select '12.4' from src limit 1 union all select 'abcd' from src limit 1) s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@ppr_test@ds=12.4
+POSTHOOK: query: insert overwrite table ppr_test partition(ds = '12.4') select * from (select '12.4' from src limit 1 union all select 'abcd' from src limit 1) s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@ppr_test@ds=12.4
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION []
+PREHOOK: query: insert overwrite table ppr_test partition(ds = '12:4') select * from (select '12:4' from src limit 1 union all select 'abcd' from src limit 1) s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@ppr_test@ds=12%3A4
+POSTHOOK: query: insert overwrite table ppr_test partition(ds = '12:4') select * from (select '12:4' from src limit 1 union all select 'abcd' from src limit 1) s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@ppr_test@ds=12%3A4
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION []
+PREHOOK: query: insert overwrite table ppr_test partition(ds = '12%4') select * from (select '12%4' from src limit 1 union all select 'abcd' from src limit 1) s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@ppr_test@ds=12%254
+POSTHOOK: query: insert overwrite table ppr_test partition(ds = '12%4') select * from (select '12%4' from src limit 1 union all select 'abcd' from src limit 1) s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@ppr_test@ds=12%254
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION []
+PREHOOK: query: insert overwrite table ppr_test partition(ds = '12*4') select * from (select '12*4' from src limit 1 union all select 'abcd' from src limit 1) s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@ppr_test@ds=12%2A4
+POSTHOOK: query: insert overwrite table ppr_test partition(ds = '12*4') select * from (select '12*4' from src limit 1 union all select 'abcd' from src limit 1) s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@ppr_test@ds=12%2A4
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION []
+PREHOOK: query: select * from ppr_test where ds = '1234'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ppr_test@ds=1234
+PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-53_994_7562932232779987377/-mr-10000
+POSTHOOK: query: select * from ppr_test where ds = '1234'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ppr_test@ds=1234
+POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-53_994_7562932232779987377/-mr-10000
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION []
+abcd 1234
+1234 1234
+PREHOOK: query: select * from ppr_test where ds = '1224'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ppr_test@ds=1224
+PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-54_581_2505144516732828104/-mr-10000
+POSTHOOK: query: select * from ppr_test where ds = '1224'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ppr_test@ds=1224
+POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-54_581_2505144516732828104/-mr-10000
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION []
+abcd 1224
+1224 1224
+PREHOOK: query: select * from ppr_test where ds = '1214'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ppr_test@ds=1214
+PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-54_853_7951716559154890872/-mr-10000
+POSTHOOK: query: select * from ppr_test where ds = '1214'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ppr_test@ds=1214
+POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-54_853_7951716559154890872/-mr-10000
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION []
+abcd 1214
+1214 1214
+PREHOOK: query: select * from ppr_test where ds = '12.4'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ppr_test@ds=12.4
+PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-55_133_1358900232060245493/-mr-10000
+POSTHOOK: query: select * from ppr_test where ds = '12.4'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ppr_test@ds=12.4
+POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-55_133_1358900232060245493/-mr-10000
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION []
+abcd 12.4
+12.4 12.4
+PREHOOK: query: select * from ppr_test where ds = '12+4'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ppr_test@ds=12+4
+PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-55_396_1886989364983202450/-mr-10000
+POSTHOOK: query: select * from ppr_test where ds = '12+4'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ppr_test@ds=12+4
+POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-55_396_1886989364983202450/-mr-10000
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION []
+abcd 12+4
+12+4 12+4
+PREHOOK: query: select * from ppr_test where ds = '12:4'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ppr_test@ds=12%3A4
+PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-55_675_1879183848037008858/-mr-10000
+POSTHOOK: query: select * from ppr_test where ds = '12:4'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ppr_test@ds=12%3A4
+POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-55_675_1879183848037008858/-mr-10000
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION []
+abcd 12:4
+12:4 12:4
+PREHOOK: query: select * from ppr_test where ds = '12%4'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ppr_test@ds=12%254
+PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-55_947_7257143082337477962/-mr-10000
+POSTHOOK: query: select * from ppr_test where ds = '12%4'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ppr_test@ds=12%254
+POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-55_947_7257143082337477962/-mr-10000
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION []
+abcd 12%4
+12%4 12%4
+PREHOOK: query: select * from ppr_test where ds = '12*4'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ppr_test@ds=12%2A4
+PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-56_206_658052552869274889/-mr-10000
+POSTHOOK: query: select * from ppr_test where ds = '12*4'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ppr_test@ds=12%2A4
+POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-56_206_658052552869274889/-mr-10000
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION []
+abcd 12*4
+12*4 12*4
+PREHOOK: query: select * from ppr_test where ds = '12.*4'
+PREHOOK: type: QUERY
+PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-56_473_2360145199088935052/-mr-10000
+POSTHOOK: query: select * from ppr_test where ds = '12.*4'
+POSTHOOK: type: QUERY
+POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-56_473_2360145199088935052/-mr-10000
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION []
+PREHOOK: query: select * from ppr_test where ds = '1234' and key = '1234'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ppr_test@ds=1234
+PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-56_533_2669295483949497025/-mr-10000
+POSTHOOK: query: select * from ppr_test where ds = '1234' and key = '1234'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ppr_test@ds=1234
+POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-56_533_2669295483949497025/-mr-10000
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION []
+1234 1234
+PREHOOK: query: select * from ppr_test where ds = '1224' and key = '1224'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ppr_test@ds=1224
+PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-59_506_7569310480938065041/-mr-10000
+POSTHOOK: query: select * from ppr_test where ds = '1224' and key = '1224'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ppr_test@ds=1224
+POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-59_506_7569310480938065041/-mr-10000
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION []
+1224 1224
+PREHOOK: query: select * from ppr_test where ds = '1214' and key = '1214'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ppr_test@ds=1214
+PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-43-02_471_7943670690328696677/-mr-10000
+POSTHOOK: query: select * from ppr_test where ds = '1214' and key = '1214'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ppr_test@ds=1214
+POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-43-02_471_7943670690328696677/-mr-10000
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION []
+1214 1214
+PREHOOK: query: select * from ppr_test where ds = '12.4' and key = '12.4'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ppr_test@ds=12.4
+PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-43-05_426_6599608597912720244/-mr-10000
+POSTHOOK: query: select * from ppr_test where ds = '12.4' and key = '12.4'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ppr_test@ds=12.4
+POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-43-05_426_6599608597912720244/-mr-10000
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION []
+12.4 12.4
+PREHOOK: query: select * from ppr_test where ds = '12+4' and key = '12+4'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ppr_test@ds=12+4
+PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-43-08_327_2168484222242026552/-mr-10000
+POSTHOOK: query: select * from ppr_test where ds = '12+4' and key = '12+4'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ppr_test@ds=12+4
+POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-43-08_327_2168484222242026552/-mr-10000
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION []
+12+4 12+4
+PREHOOK: query: select * from ppr_test where ds = '12:4' and key = '12:4'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ppr_test@ds=12%3A4
+PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-43-11_361_8648952567109428131/-mr-10000
+POSTHOOK: query: select * from ppr_test where ds = '12:4' and key = '12:4'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ppr_test@ds=12%3A4
+POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-43-11_361_8648952567109428131/-mr-10000
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION []
+12:4 12:4
+PREHOOK: query: select * from ppr_test where ds = '12%4' and key = '12%4'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ppr_test@ds=12%254
+PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-43-14_254_1733336023126589571/-mr-10000
+POSTHOOK: query: select * from ppr_test where ds = '12%4' and key = '12%4'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ppr_test@ds=12%254
+POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-43-14_254_1733336023126589571/-mr-10000
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION []
+12%4 12%4
+PREHOOK: query: select * from ppr_test where ds = '12*4' and key = '12*4'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ppr_test@ds=12%2A4
+PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-43-17_286_8476862252367771252/-mr-10000
+POSTHOOK: query: select * from ppr_test where ds = '12*4' and key = '12*4'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ppr_test@ds=12%2A4
+POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-43-17_286_8476862252367771252/-mr-10000
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION []
+12*4 12*4
Added: hive/trunk/ql/src/test/results/clientpositive/ppr_pushdown2.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/ppr_pushdown2.q.out?rev=1084240&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/ppr_pushdown2.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/ppr_pushdown2.q.out Tue Mar 22 16:33:10 2011
@@ -0,0 +1,274 @@
+PREHOOK: query: create table ppr_test (key string) partitioned by (ds string)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table ppr_test (key string) partitioned by (ds string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@ppr_test
+PREHOOK: query: insert overwrite table ppr_test partition(ds='2') select '2' from src limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@ppr_test@ds=2
+POSTHOOK: query: insert overwrite table ppr_test partition(ds='2') select '2' from src limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@ppr_test@ds=2
+POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE []
+PREHOOK: query: insert overwrite table ppr_test partition(ds='22') select '22' from src limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@ppr_test@ds=22
+POSTHOOK: query: insert overwrite table ppr_test partition(ds='22') select '22' from src limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@ppr_test@ds=22
+POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE []
+PREHOOK: query: select * from ppr_test where ds = '2'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ppr_test@ds=2
+PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-37-54_191_4798709120900823624/-mr-10000
+POSTHOOK: query: select * from ppr_test where ds = '2'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ppr_test@ds=2
+POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-37-54_191_4798709120900823624/-mr-10000
+POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE []
+2 2
+PREHOOK: query: select * from ppr_test where ds = '22'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ppr_test@ds=22
+PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-37-54_756_2735541929297276723/-mr-10000
+POSTHOOK: query: select * from ppr_test where ds = '22'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ppr_test@ds=22
+POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-37-54_756_2735541929297276723/-mr-10000
+POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE []
+22 22
+PREHOOK: query: create table ppr_test2 (key string) partitioned by (ds string, s string)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table ppr_test2 (key string) partitioned by (ds string, s string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@ppr_test2
+POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE []
+PREHOOK: query: insert overwrite table ppr_test2 partition(ds='1', s='2') select '1' from src limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@ppr_test2@ds=1/s=2
+POSTHOOK: query: insert overwrite table ppr_test2 partition(ds='1', s='2') select '1' from src limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@ppr_test2@ds=1/s=2
+POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test2 PARTITION(ds=1,s=2).key SIMPLE []
+PREHOOK: query: insert overwrite table ppr_test2 partition(ds='2', s='1') select '2' from src limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@ppr_test2@ds=2/s=1
+POSTHOOK: query: insert overwrite table ppr_test2 partition(ds='2', s='1') select '2' from src limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@ppr_test2@ds=2/s=1
+POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test2 PARTITION(ds=1,s=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test2 PARTITION(ds=2,s=1).key SIMPLE []
+PREHOOK: query: select * from ppr_test2 where s = '1'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ppr_test2@ds=2/s=1
+PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-03_996_5327659435572703549/-mr-10000
+POSTHOOK: query: select * from ppr_test2 where s = '1'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ppr_test2@ds=2/s=1
+POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-03_996_5327659435572703549/-mr-10000
+POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test2 PARTITION(ds=1,s=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test2 PARTITION(ds=2,s=1).key SIMPLE []
+2 2 1
+PREHOOK: query: select * from ppr_test2 where ds = '1'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ppr_test2@ds=1/s=2
+PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-04_281_8196775271555582839/-mr-10000
+POSTHOOK: query: select * from ppr_test2 where ds = '1'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ppr_test2@ds=1/s=2
+POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-04_281_8196775271555582839/-mr-10000
+POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test2 PARTITION(ds=1,s=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test2 PARTITION(ds=2,s=1).key SIMPLE []
+1 1 2
+PREHOOK: query: create table ppr_test3 (key string) partitioned by (col string, ol string, l string)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table ppr_test3 (key string) partitioned by (col string, ol string, l string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@ppr_test3
+POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test2 PARTITION(ds=1,s=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test2 PARTITION(ds=2,s=1).key SIMPLE []
+PREHOOK: query: insert overwrite table ppr_test3 partition(col='1', ol='2', l = '3') select '1' from src limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@ppr_test3@col=1/ol=2/l=3
+POSTHOOK: query: insert overwrite table ppr_test3 partition(col='1', ol='2', l = '3') select '1' from src limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@ppr_test3@col=1/ol=2/l=3
+POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test2 PARTITION(ds=1,s=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test2 PARTITION(ds=2,s=1).key SIMPLE []
+POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=3).key SIMPLE []
+PREHOOK: query: insert overwrite table ppr_test3 partition(col='1', ol='1', l = '2') select '2' from src limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@ppr_test3@col=1/ol=1/l=2
+POSTHOOK: query: insert overwrite table ppr_test3 partition(col='1', ol='1', l = '2') select '2' from src limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@ppr_test3@col=1/ol=1/l=2
+POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test2 PARTITION(ds=1,s=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test2 PARTITION(ds=2,s=1).key SIMPLE []
+POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=1,l=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=3).key SIMPLE []
+PREHOOK: query: insert overwrite table ppr_test3 partition(col='1', ol='2', l = '1') select '3' from src limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@ppr_test3@col=1/ol=2/l=1
+POSTHOOK: query: insert overwrite table ppr_test3 partition(col='1', ol='2', l = '1') select '3' from src limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@ppr_test3@col=1/ol=2/l=1
+POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test2 PARTITION(ds=1,s=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test2 PARTITION(ds=2,s=1).key SIMPLE []
+POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=1,l=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=1).key SIMPLE []
+POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=3).key SIMPLE []
+PREHOOK: query: select * from ppr_test3 where l = '1'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ppr_test3@col=1/ol=2/l=1
+PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-17_246_6425170733915100693/-mr-10000
+POSTHOOK: query: select * from ppr_test3 where l = '1'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ppr_test3@col=1/ol=2/l=1
+POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-17_246_6425170733915100693/-mr-10000
+POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test2 PARTITION(ds=1,s=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test2 PARTITION(ds=2,s=1).key SIMPLE []
+POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=1,l=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=1).key SIMPLE []
+POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=3).key SIMPLE []
+3 1 2 1
+PREHOOK: query: select * from ppr_test3 where l = '2'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ppr_test3@col=1/ol=1/l=2
+PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-17_523_447214062612676127/-mr-10000
+POSTHOOK: query: select * from ppr_test3 where l = '2'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ppr_test3@col=1/ol=1/l=2
+POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-17_523_447214062612676127/-mr-10000
+POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test2 PARTITION(ds=1,s=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test2 PARTITION(ds=2,s=1).key SIMPLE []
+POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=1,l=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=1).key SIMPLE []
+POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=3).key SIMPLE []
+2 1 1 2
+PREHOOK: query: select * from ppr_test3 where ol = '1'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ppr_test3@col=1/ol=1/l=2
+PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-17_813_6088158785291097781/-mr-10000
+POSTHOOK: query: select * from ppr_test3 where ol = '1'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ppr_test3@col=1/ol=1/l=2
+POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-17_813_6088158785291097781/-mr-10000
+POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test2 PARTITION(ds=1,s=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test2 PARTITION(ds=2,s=1).key SIMPLE []
+POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=1,l=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=1).key SIMPLE []
+POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=3).key SIMPLE []
+2 1 1 2
+PREHOOK: query: select * from ppr_test3 where ol = '2'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ppr_test3@col=1/ol=2/l=1
+PREHOOK: Input: default@ppr_test3@col=1/ol=2/l=3
+PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-18_142_6933403927924739851/-mr-10000
+POSTHOOK: query: select * from ppr_test3 where ol = '2'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ppr_test3@col=1/ol=2/l=1
+POSTHOOK: Input: default@ppr_test3@col=1/ol=2/l=3
+POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-18_142_6933403927924739851/-mr-10000
+POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test2 PARTITION(ds=1,s=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test2 PARTITION(ds=2,s=1).key SIMPLE []
+POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=1,l=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=1).key SIMPLE []
+POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=3).key SIMPLE []
+3 1 2 1
+1 1 2 3
+PREHOOK: query: select * from ppr_test3 where col = '1'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ppr_test3@col=1/ol=1/l=2
+PREHOOK: Input: default@ppr_test3@col=1/ol=2/l=1
+PREHOOK: Input: default@ppr_test3@col=1/ol=2/l=3
+PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-18_602_2485861984482150436/-mr-10000
+POSTHOOK: query: select * from ppr_test3 where col = '1'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ppr_test3@col=1/ol=1/l=2
+POSTHOOK: Input: default@ppr_test3@col=1/ol=2/l=1
+POSTHOOK: Input: default@ppr_test3@col=1/ol=2/l=3
+POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-18_602_2485861984482150436/-mr-10000
+POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test2 PARTITION(ds=1,s=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test2 PARTITION(ds=2,s=1).key SIMPLE []
+POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=1,l=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=1).key SIMPLE []
+POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=3).key SIMPLE []
+2 1 1 2
+3 1 2 1
+1 1 2 3
+PREHOOK: query: select * from ppr_test3 where ol = '2' and l = '1'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ppr_test3@col=1/ol=2/l=1
+PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-19_214_5779339277041051549/-mr-10000
+POSTHOOK: query: select * from ppr_test3 where ol = '2' and l = '1'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ppr_test3@col=1/ol=2/l=1
+POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-19_214_5779339277041051549/-mr-10000
+POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test2 PARTITION(ds=1,s=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test2 PARTITION(ds=2,s=1).key SIMPLE []
+POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=1,l=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=1).key SIMPLE []
+POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=3).key SIMPLE []
+3 1 2 1
+PREHOOK: query: select * from ppr_test3 where col='1' and ol = '2' and l = '1'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ppr_test3@col=1/ol=2/l=1
+PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-19_526_2052468333243542612/-mr-10000
+POSTHOOK: query: select * from ppr_test3 where col='1' and ol = '2' and l = '1'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ppr_test3@col=1/ol=2/l=1
+POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-19_526_2052468333243542612/-mr-10000
+POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE []
+POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test2 PARTITION(ds=1,s=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test2 PARTITION(ds=2,s=1).key SIMPLE []
+POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=1,l=2).key SIMPLE []
+POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=1).key SIMPLE []
+POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=3).key SIMPLE []
+3 1 2 1