You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jv...@apache.org on 2011/09/03 07:36:54 UTC
svn commit: r1164804 [1/6] - in /hive/trunk:
common/src/java/org/apache/hadoop/hive/conf/ conf/
ql/src/java/org/apache/hadoop/hive/ql/optimizer/
ql/src/java/org/apache/hadoop/hive/ql/ppd/
ql/src/test/results/clientpositive/ ql/src/test/results/compiler...
Author: jvs
Date: Sat Sep 3 05:36:52 2011
New Revision: 1164804
URL: http://svn.apache.org/viewvc?rev=1164804&view=rev
Log:
HIVE-1989. Recognize transitivity of predicates on join keys
(Charles Chen via jvs)
Modified:
hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/trunk/conf/hive-default.xml
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java
hive/trunk/ql/src/test/results/clientpositive/auto_join16.q.out
hive/trunk/ql/src/test/results/clientpositive/auto_join8.q.out
hive/trunk/ql/src/test/results/clientpositive/cluster.q.out
hive/trunk/ql/src/test/results/clientpositive/index_auto_mult_tables.q.out
hive/trunk/ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out
hive/trunk/ql/src/test/results/clientpositive/join16.q.out
hive/trunk/ql/src/test/results/clientpositive/join38.q.out
hive/trunk/ql/src/test/results/clientpositive/join8.q.out
hive/trunk/ql/src/test/results/clientpositive/louter_join_ppr.q.out
hive/trunk/ql/src/test/results/clientpositive/outer_join_ppr.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_clusterby.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_gby_join.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_join.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_join2.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_join3.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_outer_join1.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_outer_join2.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_outer_join3.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_outer_join4.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_outer_join5.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_udf_case.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_union.q.out
hive/trunk/ql/src/test/results/clientpositive/router_join_ppr.q.out
hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin9.q.out
hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_6.q.out
hive/trunk/ql/src/test/results/clientpositive/udf_named_struct.q.out
hive/trunk/ql/src/test/results/clientpositive/union22.q.out
hive/trunk/ql/src/test/results/compiler/plan/join1.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join2.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join3.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join8.q.xml
hive/trunk/ql/src/test/results/compiler/plan/sample1.q.xml
hive/trunk/ql/src/test/results/compiler/plan/sample2.q.xml
hive/trunk/ql/src/test/results/compiler/plan/sample3.q.xml
hive/trunk/ql/src/test/results/compiler/plan/sample4.q.xml
hive/trunk/ql/src/test/results/compiler/plan/sample5.q.xml
hive/trunk/ql/src/test/results/compiler/plan/sample6.q.xml
hive/trunk/ql/src/test/results/compiler/plan/sample7.q.xml
hive/trunk/ql/src/test/results/compiler/plan/subq.q.xml
hive/trunk/ql/src/test/results/compiler/plan/udf1.q.xml
hive/trunk/ql/src/test/results/compiler/plan/udf4.q.xml
hive/trunk/ql/src/test/results/compiler/plan/udf6.q.xml
hive/trunk/ql/src/test/results/compiler/plan/udf_case.q.xml
hive/trunk/ql/src/test/results/compiler/plan/udf_when.q.xml
hive/trunk/ql/src/test/results/compiler/plan/union.q.xml
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1164804&r1=1164803&r2=1164804&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Sat Sep 3 05:36:52 2011
@@ -373,6 +373,7 @@ public class HiveConf extends Configurat
HIVEOPTINDEXFILTER("hive.optimize.index.filter", false), // automatically use indexes
HIVEINDEXAUTOUPDATE("hive.optimize.index.autoupdate", false), //automatically update stale indexes
HIVEOPTPPD("hive.optimize.ppd", true), // predicate pushdown
+ HIVEPPDRECOGNIZETRANSITIVITY("hive.ppd.recognizetransivity", true), // predicate pushdown
HIVEPPDREMOVEDUPLICATEFILTERS("hive.ppd.remove.duplicatefilters", true),
// push predicates down to storage handlers
HIVEOPTPPD_STORAGE("hive.optimize.ppd.storage", true),
Modified: hive/trunk/conf/hive-default.xml
URL: http://svn.apache.org/viewvc/hive/trunk/conf/hive-default.xml?rev=1164804&r1=1164803&r2=1164804&view=diff
==============================================================================
--- hive/trunk/conf/hive-default.xml (original)
+++ hive/trunk/conf/hive-default.xml Sat Sep 3 05:36:52 2011
@@ -367,6 +367,12 @@
</property>
<property>
+ <name>hive.ppd.recognizetransivity</name>
+ <value>true</value>
+ <description>Whether to transitively replicate predicate filters over equijoin conditions.</description>
+</property>
+
+<property>
<name>hive.optimize.pruner</name>
<value>true</value>
<description>Whether to enable the new partition pruner which depends on predicate pushdown. If this is disabled,
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java?rev=1164804&r1=1164803&r2=1164804&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java Sat Sep 3 05:36:52 2011
@@ -508,6 +508,8 @@ public final class ColumnPrunerProcFacto
String[] nm = parResover.reverseLookup(outputCol);
ColumnInfo colInfo = oldRR.get(nm[0], nm[1]);
if (colInfo != null) {
+ String internalName=colInfo.getInternalName();
+ newMap.put(internalName, oldMap.get(internalName));
newRR.put(nm[0], nm[1], colInfo);
}
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java?rev=1164804&r1=1164803&r2=1164804&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java Sat Sep 3 05:36:52 2011
@@ -299,14 +299,9 @@ public final class ExprWalkerProcFactory
*/
private static void extractFinalCandidates(ExprNodeDesc expr,
ExprWalkerInfo ctx, HiveConf conf) {
- if (ctx.isCandidate(expr)) {
- ctx.addFinalCandidate(expr);
- return;
- } else if (!FunctionRegistry.isOpAnd(expr) &&
- HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVEPPDREMOVEDUPLICATEFILTERS)) {
- ctx.addNonFinalCandidate(expr);
- }
-
+ // We decompose an AND expression into its parts before checking if the
+ // entire expression is a candidate because each part may be a candidate
+ // for replicating transitively over an equijoin condition.
if (FunctionRegistry.isOpAnd(expr)) {
// If the operator is AND, we need to determine if any of the children are
// final candidates.
@@ -321,6 +316,15 @@ public final class ExprWalkerProcFactory
extractFinalCandidates((ExprNodeDesc) expr.getChildren().get(i),
ctx, conf);
}
+ return;
+ }
+
+ if (ctx.isCandidate(expr)) {
+ ctx.addFinalCandidate(expr);
+ return;
+ } else if (!FunctionRegistry.isOpAnd(expr) &&
+ HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVEPPDREMOVEDUPLICATEFILTERS)) {
+ ctx.addNonFinalCandidate(expr);
}
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java?rev=1164804&r1=1164803&r2=1164804&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java Sat Sep 3 05:36:52 2011
@@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.ppd;
import java.io.Serializable;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
@@ -30,6 +31,7 @@ import java.util.Stack;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.FilterOperator;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.JoinOperator;
@@ -44,9 +46,13 @@ import org.apache.hadoop.hive.ql.lib.Nod
import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler;
import org.apache.hadoop.hive.ql.metadata.HiveStoragePredicateHandler;
import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
+import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
+import org.apache.hadoop.hive.ql.parse.HiveParser;
import org.apache.hadoop.hive.ql.parse.OpParseContext;
import org.apache.hadoop.hive.ql.parse.RowResolver;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.plan.FilterDesc;
@@ -248,6 +254,10 @@ public final class OpProcFactory {
prunePreds.getFinalCandidates().remove(alias);
}
if (HiveConf.getBoolVar(owi.getParseContext().getConf(),
+ HiveConf.ConfVars.HIVEPPDRECOGNIZETRANSITIVITY)) {
+ applyFilterTransitivity((JoinOperator) nd, owi);
+ }
+ if (HiveConf.getBoolVar(owi.getParseContext().getConf(),
HiveConf.ConfVars.HIVEPPDREMOVEDUPLICATEFILTERS)) {
// Here, we add all the "non-final candidiates", ie. the predicates
// rejected from pushdown through this operator to unpushedPreds
@@ -269,6 +279,170 @@ public final class OpProcFactory {
}
/**
+ * Adds additional pushdown predicates for a join operator by replicating
+ * filters transitively over all the equijoin conditions.
+ *
+ * If we have a predicate "t.col=1" and the equijoin conditions
+ * "t.col=s.col" and "t.col=u.col", we add the filters "s.col=1" and
+ * "u.col=1". Note that this does not depend on the types of joins (ie.
+ * inner, left/right/full outer) between the tables s, t and u because if
+ * a predicate, eg. "t.col=1" is present in getFinalCandidates() at this
+ * point, we have already verified that it can be pushed down, so any rows
+ * emitted must satisfy s.col=t.col=u.col=1 and replicating the filters
+ * like this is ok.
+ */
+ private void applyFilterTransitivity(JoinOperator nd, OpWalkerInfo owi)
+ throws SemanticException {
+ ExprWalkerInfo prunePreds =
+ owi.getPrunedPreds((Operator<? extends Serializable>) nd);
+ if (prunePreds != null) {
+ // We want to use the row resolvers of the parents of the join op
+ // because the rowresolver refers to the output columns of an operator
+ // and the filters at this point refer to the input columns of the join
+ // operator.
+ Map<String, RowResolver> aliasToRR =
+ new HashMap<String, RowResolver>();
+ for (Operator<? extends Serializable> o : (nd).getParentOperators()) {
+ for (String alias : owi.getRowResolver(o).getTableNames()){
+ aliasToRR.put(alias, owi.getRowResolver(o));
+ }
+ }
+
+ // eqExpressions is a list of ArrayList<ASTNode>'s, one for each table
+ // in the join. Then for each i, j and k, the join condition is that
+ // eqExpressions[i][k]=eqExpressions[j][k] (*) (ie. the columns referenced
+ // by the corresponding ASTNodes are equal). For example, if the query
+ // was SELECT * FROM a join b on a.col=b.col and a.col2=b.col2 left
+ // outer join c on b.col=c.col and b.col2=c.col2 WHERE c.col=1,
+ // eqExpressions would be [[a.col1, a.col2], [b.col1, b.col2],
+ // [c.col1, c.col2]].
+ //
+ // numEqualities is the number of equal columns in each equality
+ // "chain" and numColumns is the number of such chains.
+ //
+ // Note that (*) is guaranteed to be true for the
+ // join operator: if the equijoin condititions can't be expressed in
+ // these equal-length lists of equal columns (for example if we had the
+ // query SELECT * FROM a join b on a.col=b.col and a.col2=b.col2 left
+ // outer join c on b.col=c.col), more than one join operator is used.
+ ArrayList<ArrayList<ASTNode>> eqExpressions =
+ owi.getParseContext().getJoinContext().get(nd).getExpressions();
+ int numColumns = eqExpressions.size();
+ int numEqualities = eqExpressions.get(0).size();
+
+ // joins[i] is the join between table i and i+1 in the JoinOperator
+ JoinCondDesc[] joins = (nd).getConf().getConds();
+
+ // oldFilters contains the filters to be pushed down
+ Map<String, List<ExprNodeDesc>> oldFilters =
+ prunePreds.getFinalCandidates();
+ Map<String, List<ExprNodeDesc>> newFilters =
+ new HashMap<String, List<ExprNodeDesc>>();
+
+ // We loop through for each chain of equalities
+ for (int i=0; i<numEqualities; i++) {
+ // equalColumns[i] is the ColumnInfo corresponding to the ith term
+ // of the equality or null if the term is not a simple column
+ // reference
+ ColumnInfo[] equalColumns=new ColumnInfo[numColumns];
+ for (int j=0; j<numColumns; j++) {
+ equalColumns[j] =
+ getColumnInfoFromAST(eqExpressions.get(j).get(i), aliasToRR);
+ }
+ for (int j=0; j<numColumns; j++) {
+ for (int k=0; k<numColumns; k++) {
+ if (j != k && equalColumns[j]!= null
+ && equalColumns[k] != null) {
+ // terms j and k in the equality chain are simple columns,
+ // so we can replace instances of column j with column k
+ // in the filter and ad the replicated filter.
+ ColumnInfo left = equalColumns[j];
+ ColumnInfo right = equalColumns[k];
+ if (oldFilters.get(left.getTabAlias()) != null){
+ for (ExprNodeDesc expr :
+ oldFilters.get(left.getTabAlias())) {
+ // Only replicate the filter if there is exactly one column
+ // referenced
+ Set<String> colsreferenced =
+ new HashSet<String>(expr.getCols());
+ if (colsreferenced.size() == 1
+ && colsreferenced.contains(left.getInternalName())){
+ ExprNodeDesc newexpr = expr.clone();
+ // Replace the column reference in the filter
+ replaceColumnReference(newexpr, left.getInternalName(),
+ right.getInternalName());
+ if (newFilters.get(right.getTabAlias()) == null) {
+ newFilters.put(right.getTabAlias(),
+ new ArrayList<ExprNodeDesc>());
+ }
+ newFilters.get(right.getTabAlias()).add(newexpr);
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ for (Entry<String, List<ExprNodeDesc>> aliasToFilters
+ : newFilters.entrySet()){
+ owi.getPrunedPreds((Operator<? extends Serializable>) nd)
+ .addPushDowns(aliasToFilters.getKey(), aliasToFilters.getValue());
+ }
+ }
+ }
+
+ /**
+ * Replaces the ColumnInfo for the column referred to by an ASTNode
+ * representing "table.column" or null if the ASTNode is not in that form
+ */
+ private ColumnInfo getColumnInfoFromAST(ASTNode nd,
+ Map<String, RowResolver> aliastoRR) throws SemanticException {
+ // this bit is messy since we are parsing an ASTNode at this point
+ if (nd.getType()==HiveParser.DOT) {
+ if (nd.getChildCount()==2) {
+ if (nd.getChild(0).getType()==HiveParser.TOK_TABLE_OR_COL
+ && nd.getChild(0).getChildCount()==1
+ && nd.getChild(1).getType()==HiveParser.Identifier){
+ // We unescape the identifiers and make them lower case--this
+ // really shouldn't be done here, but getExpressions gives us the
+ // raw ASTNodes. The same thing is done in SemanticAnalyzer.
+ // parseJoinCondPopulateAlias().
+ String alias = BaseSemanticAnalyzer.unescapeIdentifier(
+ nd.getChild(0).getChild(0).getText().toLowerCase());
+ String column = BaseSemanticAnalyzer.unescapeIdentifier(
+ nd.getChild(1).getText().toLowerCase());
+ RowResolver rr=aliastoRR.get(alias);
+ if (rr == null) {
+ return null;
+ }
+ return rr.get(alias, column);
+ }
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Replaces all instances of oldColumn with newColumn in the
+ * ExprColumnDesc's of the ExprNodeDesc
+ */
+ private void replaceColumnReference(ExprNodeDesc expr,
+ String oldColumn, String newColumn) {
+ if (expr instanceof ExprNodeColumnDesc) {
+ if (((ExprNodeColumnDesc) expr).getColumn().equals(oldColumn)){
+ ((ExprNodeColumnDesc) expr).setColumn(newColumn);
+ }
+ }
+
+ if (expr.getChildren() != null){
+ for (ExprNodeDesc childexpr : expr.getChildren()) {
+ replaceColumnReference(childexpr, oldColumn, newColumn);
+ }
+ }
+ }
+
+ /**
* Figures out the aliases for whom it is safe to push predicates based on
* ANSI SQL semantics. The join conditions are left associative so "a
* RIGHT OUTER JOIN b LEFT OUTER JOIN c INNER JOIN d" is interpreted as
Modified: hive/trunk/ql/src/test/results/clientpositive/auto_join16.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/auto_join16.q.out?rev=1164804&r1=1164803&r2=1164804&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/auto_join16.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/auto_join16.q.out Sat Sep 3 05:36:52 2011
@@ -63,7 +63,7 @@ STAGE PLANS:
alias: a
Filter Operator
predicate:
- expr: ((key > 10) and (key > 20))
+ expr: (((key > 10) and (key > 20)) and (value < 200))
type: boolean
Select Operator
expressions:
@@ -109,7 +109,7 @@ STAGE PLANS:
Stage: Stage-2
Map Reduce
Alias -> Map Operator Tree:
- file:/tmp/sdong/hive_2011-02-10_01-14-34_877_5596761155901891451/-mr-10002
+ file:/tmp/charleschen/hive_2011-08-04_13-40-09_960_3421937651195483585/-mr-10002
Reduce Output Operator
sort order:
tag: -1
@@ -147,7 +147,7 @@ STAGE PLANS:
alias: a
Filter Operator
predicate:
- expr: ((key > 10) and (key > 20))
+ expr: (((key > 10) and (key > 20)) and (value < 200))
type: boolean
Select Operator
expressions:
@@ -218,7 +218,7 @@ STAGE PLANS:
alias: a
Filter Operator
predicate:
- expr: ((key > 10) and (key > 20))
+ expr: (((key > 10) and (key > 20)) and (value < 200))
type: boolean
Select Operator
expressions:
@@ -308,7 +308,7 @@ ON (subq.key = tab.key and subq.key > 20
where tab.value < 200
PREHOOK: type: QUERY
PREHOOK: Input: default@src
-PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-14-35_133_7289459318323276940/-mr-10000
+PREHOOK: Output: file:/tmp/charleschen/hive_2011-08-04_13-40-10_383_7393462303376577042/-mr-10000
POSTHOOK: query: SELECT sum(hash(subq.key, tab.value))
FROM
(select a.key, a.value from src a where a.key > 10 ) subq
@@ -317,5 +317,5 @@ ON (subq.key = tab.key and subq.key > 20
where tab.value < 200
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
-POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-14-35_133_7289459318323276940/-mr-10000
+POSTHOOK: Output: file:/tmp/charleschen/hive_2011-08-04_13-40-10_383_7393462303376577042/-mr-10000
NULL
Modified: hive/trunk/ql/src/test/results/clientpositive/auto_join8.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/auto_join8.q.out?rev=1164804&r1=1164803&r2=1164804&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/auto_join8.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/auto_join8.q.out Sat Sep 3 05:36:52 2011
@@ -60,7 +60,7 @@ STAGE PLANS:
alias: src2
Filter Operator
predicate:
- expr: ((key > 15) and (key < 25))
+ expr: (((key > 15) and (key < 25)) and key is not null)
type: boolean
Select Operator
expressions:
@@ -205,7 +205,7 @@ STAGE PLANS:
alias: src2
Filter Operator
predicate:
- expr: ((key > 15) and (key < 25))
+ expr: (((key > 15) and (key < 25)) and key is not null)
type: boolean
Select Operator
expressions:
@@ -323,11 +323,11 @@ POSTHOOK: Lineage: dest1.c4 SIMPLE [(src
PREHOOK: query: SELECT sum(hash(dest1.c1,dest1.c2,dest1.c3,dest1.c4)) FROM dest1
PREHOOK: type: QUERY
PREHOOK: Input: default@dest1
-PREHOOK: Output: file:/var/folders/nt/ng21tg0n1jl4547lw0k8lg6hq_nw87/T/charleschen/hive_2011-08-30_19-20-19_577_8461311373630191052/-mr-10000
+PREHOOK: Output: file:/tmp/charleschen/hive_2011-08-31_23-13-58_896_3042859516900336699/-mr-10000
POSTHOOK: query: SELECT sum(hash(dest1.c1,dest1.c2,dest1.c3,dest1.c4)) FROM dest1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@dest1
-POSTHOOK: Output: file:/var/folders/nt/ng21tg0n1jl4547lw0k8lg6hq_nw87/T/charleschen/hive_2011-08-30_19-20-19_577_8461311373630191052/-mr-10000
+POSTHOOK: Output: file:/tmp/charleschen/hive_2011-08-31_23-13-58_896_3042859516900336699/-mr-10000
POSTHOOK: Lineage: dest1.c1 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: dest1.c2 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: dest1.c3 EXPRESSION [(src)src2.FieldSchema(name:key, type:string, comment:default), ]
Modified: hive/trunk/ql/src/test/results/clientpositive/cluster.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/cluster.q.out?rev=1164804&r1=1164803&r2=1164804&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/cluster.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/cluster.q.out Sat Sep 3 05:36:52 2011
@@ -60,11 +60,11 @@ STAGE PLANS:
PREHOOK: query: SELECT * FROM SRC x where x.key = 10 CLUSTER BY x.key
PREHOOK: type: QUERY
PREHOOK: Input: default@src
-PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-39-00_308_3996892596871050805/-mr-10000
+PREHOOK: Output: file:/tmp/charleschen/hive_2011-08-04_14-05-40_010_3384539938958696218/-mr-10000
POSTHOOK: query: SELECT * FROM SRC x where x.key = 10 CLUSTER BY x.key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
-POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-39-00_308_3996892596871050805/-mr-10000
+POSTHOOK: Output: file:/tmp/charleschen/hive_2011-08-04_14-05-40_010_3384539938958696218/-mr-10000
10 val_10
PREHOOK: query: EXPLAIN
SELECT * FROM SRC x where x.key = 20 CLUSTER BY key
@@ -128,11 +128,11 @@ STAGE PLANS:
PREHOOK: query: SELECT * FROM SRC x where x.key = 20 CLUSTER BY key
PREHOOK: type: QUERY
PREHOOK: Input: default@src
-PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-39-05_095_3989764442537175585/-mr-10000
+PREHOOK: Output: file:/tmp/charleschen/hive_2011-08-04_14-05-43_118_4008555077620962492/-mr-10000
POSTHOOK: query: SELECT * FROM SRC x where x.key = 20 CLUSTER BY key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
-POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-39-05_095_3989764442537175585/-mr-10000
+POSTHOOK: Output: file:/tmp/charleschen/hive_2011-08-04_14-05-43_118_4008555077620962492/-mr-10000
20 val_20
PREHOOK: query: EXPLAIN
SELECT x.* FROM SRC x where x.key = 20 CLUSTER BY key
@@ -196,11 +196,11 @@ STAGE PLANS:
PREHOOK: query: SELECT x.* FROM SRC x where x.key = 20 CLUSTER BY key
PREHOOK: type: QUERY
PREHOOK: Input: default@src
-PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-39-08_623_1909163536697384354/-mr-10000
+PREHOOK: Output: file:/tmp/charleschen/hive_2011-08-04_14-05-46_242_6968529164569055456/-mr-10000
POSTHOOK: query: SELECT x.* FROM SRC x where x.key = 20 CLUSTER BY key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
-POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-39-08_623_1909163536697384354/-mr-10000
+POSTHOOK: Output: file:/tmp/charleschen/hive_2011-08-04_14-05-46_242_6968529164569055456/-mr-10000
20 val_20
PREHOOK: query: EXPLAIN
SELECT x.* FROM SRC x where x.key = 20 CLUSTER BY x.key
@@ -264,11 +264,11 @@ STAGE PLANS:
PREHOOK: query: SELECT x.* FROM SRC x where x.key = 20 CLUSTER BY x.key
PREHOOK: type: QUERY
PREHOOK: Input: default@src
-PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-39-12_155_5678606790020408085/-mr-10000
+PREHOOK: Output: file:/tmp/charleschen/hive_2011-08-04_14-05-49_297_6381899498968859135/-mr-10000
POSTHOOK: query: SELECT x.* FROM SRC x where x.key = 20 CLUSTER BY x.key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
-POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-39-12_155_5678606790020408085/-mr-10000
+POSTHOOK: Output: file:/tmp/charleschen/hive_2011-08-04_14-05-49_297_6381899498968859135/-mr-10000
20 val_20
PREHOOK: query: EXPLAIN
SELECT x.key, x.value as v1 FROM SRC x where x.key = 20 CLUSTER BY key
@@ -332,11 +332,11 @@ STAGE PLANS:
PREHOOK: query: SELECT x.key, x.value as v1 FROM SRC x where x.key = 20 CLUSTER BY key
PREHOOK: type: QUERY
PREHOOK: Input: default@src
-PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-39-15_632_3313925045921329153/-mr-10000
+PREHOOK: Output: file:/tmp/charleschen/hive_2011-08-04_14-05-52_360_3921002830592058578/-mr-10000
POSTHOOK: query: SELECT x.key, x.value as v1 FROM SRC x where x.key = 20 CLUSTER BY key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
-POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-39-15_632_3313925045921329153/-mr-10000
+POSTHOOK: Output: file:/tmp/charleschen/hive_2011-08-04_14-05-52_360_3921002830592058578/-mr-10000
20 val_20
PREHOOK: query: EXPLAIN
SELECT x.key, x.value as v1 FROM SRC x where x.key = 20 CLUSTER BY x.key
@@ -400,11 +400,11 @@ STAGE PLANS:
PREHOOK: query: SELECT x.key, x.value as v1 FROM SRC x where x.key = 20 CLUSTER BY x.key
PREHOOK: type: QUERY
PREHOOK: Input: default@src
-PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-39-19_114_1622685930107208593/-mr-10000
+PREHOOK: Output: file:/tmp/charleschen/hive_2011-08-04_14-05-55_413_1561312347468537043/-mr-10000
POSTHOOK: query: SELECT x.key, x.value as v1 FROM SRC x where x.key = 20 CLUSTER BY x.key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
-POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-39-19_114_1622685930107208593/-mr-10000
+POSTHOOK: Output: file:/tmp/charleschen/hive_2011-08-04_14-05-55_413_1561312347468537043/-mr-10000
20 val_20
PREHOOK: query: EXPLAIN
SELECT x.key, x.value as v1 FROM SRC x where x.key = 20 CLUSTER BY v1
@@ -468,11 +468,11 @@ STAGE PLANS:
PREHOOK: query: SELECT x.key, x.value as v1 FROM SRC x where x.key = 20 CLUSTER BY v1
PREHOOK: type: QUERY
PREHOOK: Input: default@src
-PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-39-22_595_5329567143269229226/-mr-10000
+PREHOOK: Output: file:/tmp/charleschen/hive_2011-08-04_14-05-58_483_4193468965645427950/-mr-10000
POSTHOOK: query: SELECT x.key, x.value as v1 FROM SRC x where x.key = 20 CLUSTER BY v1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
-POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-39-22_595_5329567143269229226/-mr-10000
+POSTHOOK: Output: file:/tmp/charleschen/hive_2011-08-04_14-05-58_483_4193468965645427950/-mr-10000
20 val_20
PREHOOK: query: EXPLAIN
SELECT y.* from (SELECT x.* FROM SRC x CLUSTER BY x.key) y where y.key = 20
@@ -543,11 +543,11 @@ STAGE PLANS:
PREHOOK: query: SELECT y.* from (SELECT x.* FROM SRC x CLUSTER BY x.key) y where y.key = 20
PREHOOK: type: QUERY
PREHOOK: Input: default@src
-PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-39-26_137_8371385798213040613/-mr-10000
+PREHOOK: Output: file:/tmp/charleschen/hive_2011-08-04_14-06-01_536_2296687090295261658/-mr-10000
POSTHOOK: query: SELECT y.* from (SELECT x.* FROM SRC x CLUSTER BY x.key) y where y.key = 20
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
-POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-39-26_137_8371385798213040613/-mr-10000
+POSTHOOK: Output: file:/tmp/charleschen/hive_2011-08-04_14-06-01_536_2296687090295261658/-mr-10000
20 val_20
PREHOOK: query: EXPLAIN
SELECT x.key, x.value as v1, y.key FROM SRC x JOIN SRC y ON (x.key = y.key) where x.key = 20 CLUSTER BY v1
@@ -591,18 +591,22 @@ STAGE PLANS:
y
TableScan
alias: y
- Reduce Output Operator
- key expressions:
- expr: key
- type: string
- sort order: +
- Map-reduce partition columns:
- expr: key
- type: string
- tag: 1
- value expressions:
- expr: key
- type: string
+ Filter Operator
+ predicate:
+ expr: (key = 20)
+ type: boolean
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 1
+ value expressions:
+ expr: key
+ type: string
Reduce Operator Tree:
Join Operator
condition map:
@@ -631,7 +635,7 @@ STAGE PLANS:
Stage: Stage-2
Map Reduce
Alias -> Map Operator Tree:
- file:/tmp/sdong/hive_2011-02-10_01-39-29_546_9044307911488476013/-mr-10002
+ file:/tmp/charleschen/hive_2011-08-04_14-06-04_609_1013539649990230513/-mr-10002
Reduce Output Operator
key expressions:
expr: _col1
@@ -665,11 +669,11 @@ STAGE PLANS:
PREHOOK: query: SELECT x.key, x.value as v1, y.key FROM SRC x JOIN SRC y ON (x.key = y.key) where x.key = 20 CLUSTER BY v1
PREHOOK: type: QUERY
PREHOOK: Input: default@src
-PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-39-29_640_7990093747029711700/-mr-10000
+PREHOOK: Output: file:/tmp/charleschen/hive_2011-08-04_14-06-04_683_8881086943921866376/-mr-10000
POSTHOOK: query: SELECT x.key, x.value as v1, y.key FROM SRC x JOIN SRC y ON (x.key = y.key) where x.key = 20 CLUSTER BY v1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
-POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-39-29_640_7990093747029711700/-mr-10000
+POSTHOOK: Output: file:/tmp/charleschen/hive_2011-08-04_14-06-04_683_8881086943921866376/-mr-10000
20 val_20 20
PREHOOK: query: EXPLAIN
SELECT x.key, x.value as v1, y.* FROM SRC x JOIN SRC y ON (x.key = y.key) where x.key = 20 CLUSTER BY v1
@@ -713,20 +717,24 @@ STAGE PLANS:
y
TableScan
alias: y
- Reduce Output Operator
- key expressions:
- expr: key
- type: string
- sort order: +
- Map-reduce partition columns:
- expr: key
- type: string
- tag: 1
- value expressions:
- expr: key
- type: string
- expr: value
- type: string
+ Filter Operator
+ predicate:
+ expr: (key = 20)
+ type: boolean
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 1
+ value expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
Reduce Operator Tree:
Join Operator
condition map:
@@ -757,7 +765,7 @@ STAGE PLANS:
Stage: Stage-2
Map Reduce
Alias -> Map Operator Tree:
- file:/tmp/sdong/hive_2011-02-10_01-39-36_146_7523047334744872007/-mr-10002
+ file:/tmp/charleschen/hive_2011-08-04_14-06-10_540_387097519944809883/-mr-10002
Reduce Output Operator
key expressions:
expr: _col1
@@ -793,11 +801,11 @@ STAGE PLANS:
PREHOOK: query: SELECT x.key, x.value as v1, y.* FROM SRC x JOIN SRC y ON (x.key = y.key) where x.key = 20 CLUSTER BY v1
PREHOOK: type: QUERY
PREHOOK: Input: default@src
-PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-39-36_271_8501113562060498197/-mr-10000
+PREHOOK: Output: file:/tmp/charleschen/hive_2011-08-04_14-06-10_620_508384745366093039/-mr-10000
POSTHOOK: query: SELECT x.key, x.value as v1, y.* FROM SRC x JOIN SRC y ON (x.key = y.key) where x.key = 20 CLUSTER BY v1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
-POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-39-36_271_8501113562060498197/-mr-10000
+POSTHOOK: Output: file:/tmp/charleschen/hive_2011-08-04_14-06-10_620_508384745366093039/-mr-10000
20 val_20 20 val_20
PREHOOK: query: EXPLAIN
SELECT x.key, x.value as v1, y.* FROM SRC x JOIN SRC y ON (x.key = y.key) where x.key = 20 CLUSTER BY x.key
@@ -841,20 +849,24 @@ STAGE PLANS:
y
TableScan
alias: y
- Reduce Output Operator
- key expressions:
- expr: key
- type: string
- sort order: +
- Map-reduce partition columns:
- expr: key
- type: string
- tag: 1
- value expressions:
- expr: key
- type: string
- expr: value
- type: string
+ Filter Operator
+ predicate:
+ expr: (key = 20)
+ type: boolean
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 1
+ value expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
Reduce Operator Tree:
Join Operator
condition map:
@@ -885,7 +897,7 @@ STAGE PLANS:
Stage: Stage-2
Map Reduce
Alias -> Map Operator Tree:
- file:/tmp/sdong/hive_2011-02-10_01-39-42_907_3531906775052670778/-mr-10002
+ file:/tmp/charleschen/hive_2011-08-04_14-06-16_477_4713776691114383621/-mr-10002
Reduce Output Operator
key expressions:
expr: _col0
@@ -921,11 +933,11 @@ STAGE PLANS:
PREHOOK: query: SELECT x.key, x.value as v1, y.* FROM SRC x JOIN SRC y ON (x.key = y.key) where x.key = 20 CLUSTER BY x.key
PREHOOK: type: QUERY
PREHOOK: Input: default@src
-PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-39-43_027_3013451718430380875/-mr-10000
+PREHOOK: Output: file:/tmp/charleschen/hive_2011-08-04_14-06-16_550_1693241149285515315/-mr-10000
POSTHOOK: query: SELECT x.key, x.value as v1, y.* FROM SRC x JOIN SRC y ON (x.key = y.key) where x.key = 20 CLUSTER BY x.key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
-POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-39-43_027_3013451718430380875/-mr-10000
+POSTHOOK: Output: file:/tmp/charleschen/hive_2011-08-04_14-06-16_550_1693241149285515315/-mr-10000
20 val_20 20 val_20
PREHOOK: query: EXPLAIN
SELECT x.key, x.value as v1, y.key as yk FROM SRC x JOIN SRC y ON (x.key = y.key) where x.key = 20 CLUSTER BY key
@@ -969,18 +981,22 @@ STAGE PLANS:
y
TableScan
alias: y
- Reduce Output Operator
- key expressions:
- expr: key
- type: string
- sort order: +
- Map-reduce partition columns:
- expr: key
- type: string
- tag: 1
- value expressions:
- expr: key
- type: string
+ Filter Operator
+ predicate:
+ expr: (key = 20)
+ type: boolean
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 1
+ value expressions:
+ expr: key
+ type: string
Reduce Operator Tree:
Join Operator
condition map:
@@ -1009,7 +1025,7 @@ STAGE PLANS:
Stage: Stage-2
Map Reduce
Alias -> Map Operator Tree:
- file:/tmp/sdong/hive_2011-02-10_01-39-49_649_313028914611607523/-mr-10002
+ file:/tmp/charleschen/hive_2011-08-04_14-06-22_495_1958918890600969598/-mr-10002
Reduce Output Operator
key expressions:
expr: _col0
@@ -1043,11 +1059,11 @@ STAGE PLANS:
PREHOOK: query: SELECT x.key, x.value as v1, y.key as yk FROM SRC x JOIN SRC y ON (x.key = y.key) where x.key = 20 CLUSTER BY key
PREHOOK: type: QUERY
PREHOOK: Input: default@src
-PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-39-49_743_5449040629326573495/-mr-10000
+PREHOOK: Output: file:/tmp/charleschen/hive_2011-08-04_14-06-22_565_5589926363417940722/-mr-10000
POSTHOOK: query: SELECT x.key, x.value as v1, y.key as yk FROM SRC x JOIN SRC y ON (x.key = y.key) where x.key = 20 CLUSTER BY key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
-POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-39-49_743_5449040629326573495/-mr-10000
+POSTHOOK: Output: file:/tmp/charleschen/hive_2011-08-04_14-06-22_565_5589926363417940722/-mr-10000
20 val_20 20
PREHOOK: query: EXPLAIN
SELECT unioninput.*
@@ -1173,7 +1189,7 @@ FROM (
CLUSTER BY unioninput.key
PREHOOK: type: QUERY
PREHOOK: Input: default@src
-PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-39-56_398_2632059281772742928/-mr-10000
+PREHOOK: Output: file:/tmp/charleschen/hive_2011-08-04_14-06-28_484_6154449119111735745/-mr-10000
POSTHOOK: query: SELECT unioninput.*
FROM (
FROM src select src.key, src.value WHERE src.key < 100
@@ -1183,7 +1199,7 @@ FROM (
CLUSTER BY unioninput.key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
-POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_01-39-56_398_2632059281772742928/-mr-10000
+POSTHOOK: Output: file:/tmp/charleschen/hive_2011-08-04_14-06-28_484_6154449119111735745/-mr-10000
0 val_0
0 val_0
0 val_0
Modified: hive/trunk/ql/src/test/results/clientpositive/index_auto_mult_tables.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/index_auto_mult_tables.q.out?rev=1164804&r1=1164803&r2=1164804&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/index_auto_mult_tables.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/index_auto_mult_tables.q.out Sat Sep 3 05:36:52 2011
@@ -25,7 +25,7 @@ STAGE PLANS:
alias: a
Filter Operator
predicate:
- expr: ((key > 80) and (key < 100))
+ expr: ((((key > 80) and (key < 100)) and (key > 70)) and (key < 90))
type: boolean
Reduce Output Operator
key expressions:
@@ -46,7 +46,7 @@ STAGE PLANS:
alias: b
Filter Operator
predicate:
- expr: ((key > 70) and (key < 90))
+ expr: ((((key > 70) and (key < 90)) and (key > 80)) and (key < 100))
type: boolean
Reduce Output Operator
key expressions:
@@ -86,7 +86,7 @@ STAGE PLANS:
Stage: Stage-2
Map Reduce
Alias -> Map Operator Tree:
- file:/var/folders/nt/ng21tg0n1jl4547lw0k8lg6hq_nw87/T/charleschen/hive_2011-08-30_19-11-37_886_962059869422887819/-mr-10002
+ file:/tmp/charleschen/hive_2011-09-02_00-40-32_210_8369520160583859081/-mr-10002
Reduce Output Operator
key expressions:
expr: _col0
@@ -119,7 +119,7 @@ PREHOOK: Input: default@srcpart@ds=2008-
PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
-PREHOOK: Output: file:/var/folders/nt/ng21tg0n1jl4547lw0k8lg6hq_nw87/T/charleschen/hive_2011-08-30_19-11-38_313_2643938706286429021/-mr-10000
+PREHOOK: Output: file:/tmp/charleschen/hive_2011-09-02_00-40-32_298_7062969703447731515/-mr-10000
POSTHOOK: query: SELECT a.key, a.value FROM src a JOIN srcpart b ON (a.key = b.key) WHERE a.key > 80 AND a.key < 100 AND b.key > 70 AND b.key < 90 ORDER BY a.key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
@@ -127,7 +127,7 @@ POSTHOOK: Input: default@srcpart@ds=2008
POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
-POSTHOOK: Output: file:/var/folders/nt/ng21tg0n1jl4547lw0k8lg6hq_nw87/T/charleschen/hive_2011-08-30_19-11-38_313_2643938706286429021/-mr-10000
+POSTHOOK: Output: file:/tmp/charleschen/hive_2011-09-02_00-40-32_298_7062969703447731515/-mr-10000
82 val_82
82 val_82
82 val_82
@@ -260,11 +260,11 @@ STAGE PLANS:
TableScan
alias: default__srcpart_srcpart_index__
filterExpr:
- expr: (((key > 70) and (key < 90)) and (not EWAH_BITMAP_EMPTY(_bitmaps)))
+ expr: (((((key > 70) and (key < 90)) and (key > 80)) and (key < 100)) and (not EWAH_BITMAP_EMPTY(_bitmaps)))
type: boolean
Filter Operator
predicate:
- expr: (((key > 70) and (key < 90)) and (not EWAH_BITMAP_EMPTY(_bitmaps)))
+ expr: (((((key > 70) and (key < 90)) and (key > 80)) and (key < 100)) and (not EWAH_BITMAP_EMPTY(_bitmaps)))
type: boolean
Select Operator
expressions:
@@ -338,7 +338,7 @@ STAGE PLANS:
Move Operator
files:
hdfs directory: true
- destination: file:/var/folders/nt/ng21tg0n1jl4547lw0k8lg6hq_nw87/T/charleschen/hive_2011-08-30_19-12-29_480_7231326688955426901/-mr-10003
+ destination: file:/tmp/charleschen/hive_2011-09-02_00-40-58_026_6087921660802846126/-mr-10003
Stage: Stage-1
Map Reduce
@@ -347,11 +347,11 @@ STAGE PLANS:
TableScan
alias: a
filterExpr:
- expr: ((key > 80) and (key < 100))
+ expr: ((((key > 80) and (key < 100)) and (key > 70)) and (key < 90))
type: boolean
Filter Operator
predicate:
- expr: ((key > 80) and (key < 100))
+ expr: ((((key > 80) and (key < 100)) and (key > 70)) and (key < 90))
type: boolean
Reduce Output Operator
key expressions:
@@ -371,11 +371,11 @@ STAGE PLANS:
TableScan
alias: b
filterExpr:
- expr: ((key > 70) and (key < 90))
+ expr: ((((key > 70) and (key < 90)) and (key > 80)) and (key < 100))
type: boolean
Filter Operator
predicate:
- expr: ((key > 70) and (key < 90))
+ expr: ((((key > 70) and (key < 90)) and (key > 80)) and (key < 100))
type: boolean
Reduce Output Operator
key expressions:
@@ -415,7 +415,7 @@ STAGE PLANS:
Stage: Stage-2
Map Reduce
Alias -> Map Operator Tree:
- file:/var/folders/nt/ng21tg0n1jl4547lw0k8lg6hq_nw87/T/charleschen/hive_2011-08-30_19-12-29_480_7231326688955426901/-mr-10002
+ file:/tmp/charleschen/hive_2011-09-02_00-40-58_026_6087921660802846126/-mr-10002
Reduce Output Operator
key expressions:
expr: _col0
@@ -443,11 +443,11 @@ STAGE PLANS:
TableScan
alias: default__src_src_index__
filterExpr:
- expr: (((key > 80) and (key < 100)) and (not EWAH_BITMAP_EMPTY(_bitmaps)))
+ expr: (((((key > 80) and (key < 100)) and (key > 70)) and (key < 90)) and (not EWAH_BITMAP_EMPTY(_bitmaps)))
type: boolean
Filter Operator
predicate:
- expr: (((key > 80) and (key < 100)) and (not EWAH_BITMAP_EMPTY(_bitmaps)))
+ expr: (((((key > 80) and (key < 100)) and (key > 70)) and (key < 90)) and (not EWAH_BITMAP_EMPTY(_bitmaps)))
type: boolean
Select Operator
expressions:
@@ -521,7 +521,7 @@ STAGE PLANS:
Move Operator
files:
hdfs directory: true
- destination: file:/var/folders/nt/ng21tg0n1jl4547lw0k8lg6hq_nw87/T/charleschen/hive_2011-08-30_19-12-29_480_7231326688955426901/-mr-10004
+ destination: file:/tmp/charleschen/hive_2011-09-02_00-40-58_026_6087921660802846126/-mr-10004
Stage: Stage-0
Fetch Operator
@@ -540,7 +540,7 @@ PREHOOK: Input: default@srcpart@ds=2008-
PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
-PREHOOK: Output: file:/var/folders/nt/ng21tg0n1jl4547lw0k8lg6hq_nw87/T/charleschen/hive_2011-08-30_19-12-30_228_1386208254160104156/-mr-10000
+PREHOOK: Output: file:/tmp/charleschen/hive_2011-09-02_00-40-58_428_4655402597563090498/-mr-10000
POSTHOOK: query: SELECT a.key, a.value FROM src a JOIN srcpart b ON (a.key = b.key) WHERE a.key > 80 AND a.key < 100 AND b.key > 70 AND b.key < 90 ORDER BY a.key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@default__src_src_index__
@@ -553,7 +553,7 @@ POSTHOOK: Input: default@srcpart@ds=2008
POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
-POSTHOOK: Output: file:/var/folders/nt/ng21tg0n1jl4547lw0k8lg6hq_nw87/T/charleschen/hive_2011-08-30_19-12-30_228_1386208254160104156/-mr-10000
+POSTHOOK: Output: file:/tmp/charleschen/hive_2011-09-02_00-40-58_428_4655402597563090498/-mr-10000
POSTHOOK: Lineage: default__src_src_index__._bitmaps EXPRESSION [(src)src.FieldSchema(name:ROW__OFFSET__INSIDE__BLOCK, type:bigint, comment:), ]
POSTHOOK: Lineage: default__src_src_index__._bucketname SIMPLE [(src)src.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
POSTHOOK: Lineage: default__src_src_index__._offset SIMPLE [(src)src.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ]
Modified: hive/trunk/ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out?rev=1164804&r1=1164803&r2=1164804&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out Sat Sep 3 05:36:52 2011
@@ -25,7 +25,7 @@ STAGE PLANS:
alias: a
Filter Operator
predicate:
- expr: ((key > 80) and (key < 100))
+ expr: ((((key > 80) and (key < 100)) and (key > 70)) and (key < 90))
type: boolean
Reduce Output Operator
key expressions:
@@ -46,7 +46,7 @@ STAGE PLANS:
alias: b
Filter Operator
predicate:
- expr: ((key > 70) and (key < 90))
+ expr: ((((key > 70) and (key < 90)) and (key > 80)) and (key < 100))
type: boolean
Reduce Output Operator
key expressions:
@@ -86,7 +86,7 @@ STAGE PLANS:
Stage: Stage-2
Map Reduce
Alias -> Map Operator Tree:
- file:/tmp/salbiz/hive_2011-08-03_11-41-42_789_1684955694193338986/-mr-10002
+ file:/tmp/charleschen/hive_2011-09-02_00-41-12_391_4510515203785787418/-mr-10002
Reduce Output Operator
key expressions:
expr: _col0
@@ -119,7 +119,7 @@ PREHOOK: Input: default@srcpart@ds=2008-
PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
-PREHOOK: Output: file:/tmp/salbiz/hive_2011-08-03_11-41-42_970_2859185952217320568/-mr-10000
+PREHOOK: Output: file:/tmp/charleschen/hive_2011-09-02_00-41-12_469_5605211675521513930/-mr-10000
POSTHOOK: query: SELECT a.key, a.value FROM src a JOIN srcpart b ON (a.key = b.key) WHERE a.key > 80 AND a.key < 100 AND b.key > 70 AND b.key < 90 ORDER BY a.key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
@@ -127,7 +127,7 @@ POSTHOOK: Input: default@srcpart@ds=2008
POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
-POSTHOOK: Output: file:/tmp/salbiz/hive_2011-08-03_11-41-42_970_2859185952217320568/-mr-10000
+POSTHOOK: Output: file:/tmp/charleschen/hive_2011-09-02_00-41-12_469_5605211675521513930/-mr-10000
82 val_82
82 val_82
82 val_82
@@ -262,11 +262,11 @@ STAGE PLANS:
TableScan
alias: default__srcpart_srcpart_index__
filterExpr:
- expr: ((key > 70) and (key < 90))
+ expr: ((((key > 70) and (key < 90)) and (key > 80)) and (key < 100))
type: boolean
Filter Operator
predicate:
- expr: ((key > 70) and (key < 90))
+ expr: ((((key > 70) and (key < 90)) and (key > 80)) and (key < 100))
type: boolean
Select Operator
expressions:
@@ -289,13 +289,13 @@ STAGE PLANS:
Move Operator
files:
hdfs directory: true
- destination: file:/data/users/salbiz/apache-hive/build/ql/scratchdir/hive_2011-08-03_11-42-13_662_1841905286836798924/-ext-10000
+ destination: file:/data/users/charleschen/hive-trunk/build/ql/scratchdir/hive_2011-09-02_00-41-38_351_1674548679529653747/-ext-10000
Stage: Stage-4
Move Operator
files:
hdfs directory: true
- destination: file:/tmp/salbiz/hive_2011-08-03_11-42-12_993_7173516009867952993/-mr-10003
+ destination: file:/tmp/charleschen/hive_2011-09-02_00-41-38_116_119715155297408389/-mr-10003
Stage: Stage-1
Map Reduce
@@ -304,11 +304,11 @@ STAGE PLANS:
TableScan
alias: a
filterExpr:
- expr: ((key > 80) and (key < 100))
+ expr: ((((key > 80) and (key < 100)) and (key > 70)) and (key < 90))
type: boolean
Filter Operator
predicate:
- expr: ((key > 80) and (key < 100))
+ expr: ((((key > 80) and (key < 100)) and (key > 70)) and (key < 90))
type: boolean
Reduce Output Operator
key expressions:
@@ -328,11 +328,11 @@ STAGE PLANS:
TableScan
alias: b
filterExpr:
- expr: ((key > 70) and (key < 90))
+ expr: ((((key > 70) and (key < 90)) and (key > 80)) and (key < 100))
type: boolean
Filter Operator
predicate:
- expr: ((key > 70) and (key < 90))
+ expr: ((((key > 70) and (key < 90)) and (key > 80)) and (key < 100))
type: boolean
Reduce Output Operator
key expressions:
@@ -372,7 +372,7 @@ STAGE PLANS:
Stage: Stage-2
Map Reduce
Alias -> Map Operator Tree:
- file:/tmp/salbiz/hive_2011-08-03_11-42-12_993_7173516009867952993/-mr-10002
+ file:/tmp/charleschen/hive_2011-09-02_00-41-38_116_119715155297408389/-mr-10002
Reduce Output Operator
key expressions:
expr: _col0
@@ -396,7 +396,7 @@ STAGE PLANS:
Stage: Stage-6
Map Reduce
Alias -> Map Operator Tree:
- file:/data/users/salbiz/apache-hive/build/ql/scratchdir/hive_2011-08-03_11-42-13_662_1841905286836798924/-ext-10001
+ file:/data/users/charleschen/hive-trunk/build/ql/scratchdir/hive_2011-09-02_00-41-38_351_1674548679529653747/-ext-10001
File Output Operator
compressed: false
GlobalTableId: 0
@@ -411,11 +411,11 @@ STAGE PLANS:
TableScan
alias: default__src_src_index__
filterExpr:
- expr: ((key > 80) and (key < 100))
+ expr: ((((key > 80) and (key < 100)) and (key > 70)) and (key < 90))
type: boolean
Filter Operator
predicate:
- expr: ((key > 80) and (key < 100))
+ expr: ((((key > 80) and (key < 100)) and (key > 70)) and (key < 90))
type: boolean
Select Operator
expressions:
@@ -438,18 +438,18 @@ STAGE PLANS:
Move Operator
files:
hdfs directory: true
- destination: file:/data/users/salbiz/apache-hive/build/ql/scratchdir/hive_2011-08-03_11-42-13_773_6752137651963996196/-ext-10000
+ destination: file:/data/users/charleschen/hive-trunk/build/ql/scratchdir/hive_2011-09-02_00-41-38_438_6653574264861894172/-ext-10000
Stage: Stage-9
Move Operator
files:
hdfs directory: true
- destination: file:/tmp/salbiz/hive_2011-08-03_11-42-12_993_7173516009867952993/-mr-10004
+ destination: file:/tmp/charleschen/hive_2011-09-02_00-41-38_116_119715155297408389/-mr-10004
Stage: Stage-11
Map Reduce
Alias -> Map Operator Tree:
- file:/data/users/salbiz/apache-hive/build/ql/scratchdir/hive_2011-08-03_11-42-13_773_6752137651963996196/-ext-10001
+ file:/data/users/charleschen/hive-trunk/build/ql/scratchdir/hive_2011-09-02_00-41-38_438_6653574264861894172/-ext-10001
File Output Operator
compressed: false
GlobalTableId: 0
@@ -474,7 +474,7 @@ PREHOOK: Input: default@srcpart@ds=2008-
PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
-PREHOOK: Output: file:/tmp/salbiz/hive_2011-08-03_11-42-13_878_115379500838443509/-mr-10000
+PREHOOK: Output: file:/tmp/charleschen/hive_2011-09-02_00-41-38_484_8865028993150130193/-mr-10000
POSTHOOK: query: SELECT a.key, a.value FROM src a JOIN srcpart b ON (a.key = b.key) WHERE a.key > 80 AND a.key < 100 AND b.key > 70 AND b.key < 90 ORDER BY a.key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@default__src_src_index__
@@ -487,7 +487,7 @@ POSTHOOK: Input: default@srcpart@ds=2008
POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
-POSTHOOK: Output: file:/tmp/salbiz/hive_2011-08-03_11-42-13_878_115379500838443509/-mr-10000
+POSTHOOK: Output: file:/tmp/charleschen/hive_2011-09-02_00-41-38_484_8865028993150130193/-mr-10000
POSTHOOK: Lineage: default__src_src_index__._bucketname SIMPLE [(src)src.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
POSTHOOK: Lineage: default__src_src_index__._offsets EXPRESSION [(src)src.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ]
POSTHOOK: Lineage: default__src_src_index__.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
Modified: hive/trunk/ql/src/test/results/clientpositive/join16.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/join16.q.out?rev=1164804&r1=1164803&r2=1164804&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/join16.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/join16.q.out Sat Sep 3 05:36:52 2011
@@ -18,7 +18,7 @@ STAGE PLANS:
alias: a
Filter Operator
predicate:
- expr: ((key > 10) and (key > 20))
+ expr: (((key > 10) and (key > 20)) and (value < 200))
type: boolean
Select Operator
expressions:
Modified: hive/trunk/ql/src/test/results/clientpositive/join38.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/join38.q.out?rev=1164804&r1=1164803&r2=1164804&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/join38.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/join38.q.out Sat Sep 3 05:36:52 2011
@@ -26,11 +26,11 @@ POSTHOOK: Lineage: tmp.col9 EXPRESSION [
PREHOOK: query: select * from tmp
PREHOOK: type: QUERY
PREHOOK: Input: default@tmp
-PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_16-07-01_449_2704240234253573656/-mr-10000
+PREHOOK: Output: file:/tmp/charleschen/hive_2011-08-04_14-53-23_764_3301298329074450240/-mr-10000
POSTHOOK: query: select * from tmp
POSTHOOK: type: QUERY
POSTHOOK: Input: default@tmp
-POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_16-07-01_449_2704240234253573656/-mr-10000
+POSTHOOK: Output: file:/tmp/charleschen/hive_2011-08-04_14-53-23_764_3301298329074450240/-mr-10000
POSTHOOK: Lineage: tmp.col0 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: tmp.col1 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: tmp.col10 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
@@ -89,15 +89,19 @@ STAGE PLANS:
a
TableScan
alias: a
- HashTable Sink Operator
- condition expressions:
- 0 {value}
- 1 {col5} {col11}
- handleSkewJoin: false
- keys:
- 0 [Column[key]]
- 1 [Column[col11]]
- Position of Big Table: 1
+ Filter Operator
+ predicate:
+ expr: (key = 111)
+ type: boolean
+ HashTable Sink Operator
+ condition expressions:
+ 0 {value}
+ 1 {col5} {col11}
+ handleSkewJoin: false
+ keys:
+ 0 [Column[key]]
+ 1 [Column[col11]]
+ Position of Big Table: 1
Stage: Stage-1
Map Reduce
@@ -133,7 +137,7 @@ STAGE PLANS:
Stage: Stage-2
Map Reduce
Alias -> Map Operator Tree:
- file:/tmp/sdong/hive_2011-02-10_16-07-01_714_3920059174195065578/-mr-10002
+ file:/tmp/charleschen/hive_2011-08-04_14-53-23_963_3349528858179007798/-mr-10002
Select Operator
expressions:
expr: _col1
@@ -217,7 +221,7 @@ group by a.value, b.col5
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Input: default@tmp
-PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_16-07-01_819_8577478512687826786/-mr-10000
+PREHOOK: Output: file:/tmp/charleschen/hive_2011-08-04_14-53-24_044_6878564084853277625/-mr-10000
POSTHOOK: query: FROM src a JOIN tmp b ON (a.key = b.col11)
SELECT /*+ MAPJOIN(a) */ a.value, b.col5, count(1) as count
where b.col11 = 111
@@ -225,7 +229,7 @@ group by a.value, b.col5
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
POSTHOOK: Input: default@tmp
-POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_16-07-01_819_8577478512687826786/-mr-10000
+POSTHOOK: Output: file:/tmp/charleschen/hive_2011-08-04_14-53-24_044_6878564084853277625/-mr-10000
POSTHOOK: Lineage: tmp.col0 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: tmp.col1 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: tmp.col10 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
Modified: hive/trunk/ql/src/test/results/clientpositive/join8.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/join8.q.out?rev=1164804&r1=1164803&r2=1164804&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/join8.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/join8.q.out Sat Sep 3 05:36:52 2011
@@ -78,7 +78,7 @@ STAGE PLANS:
alias: src2
Filter Operator
predicate:
- expr: ((key > 15) and (key < 25))
+ expr: (((key > 15) and (key < 25)) and key is not null)
type: boolean
Select Operator
expressions:
@@ -209,11 +209,11 @@ POSTHOOK: Lineage: dest1.c4 SIMPLE [(src
PREHOOK: query: SELECT dest1.* FROM dest1
PREHOOK: type: QUERY
PREHOOK: Input: default@dest1
-PREHOOK: Output: file:/var/folders/nt/ng21tg0n1jl4547lw0k8lg6hq_nw87/T/charleschen/hive_2011-08-30_19-14-28_754_928376312270623780/-mr-10000
+PREHOOK: Output: file:/tmp/charleschen/hive_2011-09-01_00-11-39_732_8229663903723256/-mr-10000
POSTHOOK: query: SELECT dest1.* FROM dest1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@dest1
-POSTHOOK: Output: file:/var/folders/nt/ng21tg0n1jl4547lw0k8lg6hq_nw87/T/charleschen/hive_2011-08-30_19-14-28_754_928376312270623780/-mr-10000
+POSTHOOK: Output: file:/tmp/charleschen/hive_2011-09-01_00-11-39_732_8229663903723256/-mr-10000
POSTHOOK: Lineage: dest1.c1 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: dest1.c2 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: dest1.c3 EXPRESSION [(src)src2.FieldSchema(name:key, type:string, comment:default), ]