You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by zs...@apache.org on 2009/01/29 03:46:44 UTC
svn commit: r738727 [1/2] - in /hadoop/hive/trunk: ./
common/src/java/org/apache/hadoop/hive/conf/
ql/src/java/org/apache/hadoop/hive/ql/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/optimizer/ ql/src/java/org/apach...
Author: zshao
Date: Thu Jan 29 02:46:41 2009
New Revision: 738727
URL: http://svn.apache.org/viewvc?rev=738727&view=rev
Log:
HIVE-105. Automatically estimate number of required reducers. (zshao)
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/mapredWork.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/reduceSinkDesc.java
hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_limit.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby4_map.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby5_map.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input11_limit.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input14_limit.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input1_limit.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/input3_limit.q.out
hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby1.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby2.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby3.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby4.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby5.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby6.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/input20.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/input4.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/input5.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/join1.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/join2.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/join3.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/join4.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/join5.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/join6.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/join7.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/join8.q.xml
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=738727&r1=738726&r2=738727&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Thu Jan 29 02:46:41 2009
@@ -9,6 +9,8 @@
NEW FEATURES
+ HIVE-105. Automatically estimate number of required reducers. (zshao)
+
HIVE-176. Added a history log for Hive. (Suresh Anthony via athusoo)
HIVE-244. Add SQRT() UDF. (Jeff Hammerbacher via zshao)
Modified: hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=738727&r1=738726&r2=738727&view=diff
==============================================================================
--- hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Thu Jan 29 02:46:41 2009
@@ -63,6 +63,8 @@
SCRIPTERRORLIMIT("hive.exec.script.maxerrsize", 100000),
COMPRESSRESULT("hive.exec.compress.output", false),
COMPRESSINTERMEDIATE("hive.exec.compress.intermediate", false),
+ BYTESPERREDUCER("hive.exec.reducers.bytes.per.reducer", (long)(1000*1000*1000)),
+ MAXREDUCERS("hive.exec.reducers.max", 999),
// hadoop stuff
HADOOPBIN("hadoop.bin.path", System.getenv("HADOOP_HOME") + "/bin/hadoop"),
@@ -105,7 +107,7 @@
HIVEALIAS("hive.alias", ""),
HIVEMAPSIDEAGGREGATE("hive.map.aggr", "false"),
HIVEJOINEMITINTERVAL("hive.join.emit.interval", 1000),
- HIVEMAPAGGRHASHMEMORY("hive.map.aggr.hash.percentmemory", (float)0.5),
+ HIVEMAPAGGRHASHMEMORY("hive.map.aggr.hash.percentmemory", (float)0.5),
// Default file format for CREATE TABLE statement
// Options: TextFile, SequenceFile
@@ -118,43 +120,58 @@
public final String varname;
public final String defaultVal;
public final int defaultIntVal;
+ public final long defaultLongVal;
public final float defaultFloatVal;
public final Class<?> valClass;
public final boolean defaultBoolVal;
ConfVars(String varname, String defaultVal) {
this.varname = varname;
- this.defaultVal = defaultVal;
this.valClass = String.class;
+ this.defaultVal = defaultVal;
this.defaultIntVal = -1;
+ this.defaultLongVal = -1;
this.defaultFloatVal = -1;
this.defaultBoolVal = false;
}
ConfVars(String varname, int defaultIntVal) {
this.varname = varname;
+ this.valClass = Integer.class;
this.defaultVal = null;
this.defaultIntVal = defaultIntVal;
+ this.defaultLongVal = -1;
+ this.defaultFloatVal = -1;
+ this.defaultBoolVal = false;
+ }
+
+ ConfVars(String varname, long defaultLongVal) {
+ this.varname = varname;
+ this.valClass = Long.class;
+ this.defaultVal = null;
+ this.defaultIntVal = -1;
+ this.defaultLongVal = defaultLongVal;
this.defaultFloatVal = -1;
- this.valClass = Integer.class;
this.defaultBoolVal = false;
}
ConfVars(String varname, float defaultFloatVal) {
this.varname = varname;
+ this.valClass = Float.class;
this.defaultVal = null;
this.defaultIntVal = -1;
+ this.defaultLongVal = -1;
this.defaultFloatVal = defaultFloatVal;
- this.valClass = Float.class;
this.defaultBoolVal = false;
}
ConfVars(String varname, boolean defaultBoolVal) {
this.varname = varname;
+ this.valClass = Boolean.class;
this.defaultVal = null;
this.defaultIntVal = -1;
+ this.defaultLongVal = -1;
this.defaultFloatVal = -1;
- this.valClass = Boolean.class;
this.defaultBoolVal = defaultBoolVal;
}
@@ -172,6 +189,15 @@
return getIntVar(this, var);
}
+ public static long getLongVar(Configuration conf, ConfVars var) {
+ assert(var.valClass == Long.class);
+ return conf.getLong(var.varname, var.defaultLongVal);
+ }
+
+ public long getLongVar(ConfVars var) {
+ return getLongVar(this, var);
+ }
+
public static float getFloatVar(Configuration conf, ConfVars var) {
assert(var.valClass == Float.class);
return conf.getFloat(var.varname, var.defaultFloatVal);
@@ -214,7 +240,10 @@
}
}
-
+ public HiveConf() {
+ super();
+ }
+
public HiveConf(Class<?> cls) {
super();
initialize(cls);
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java?rev=738727&r1=738726&r2=738727&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java Thu Jan 29 02:46:41 2009
@@ -201,15 +201,6 @@
SessionState.get().getHiveHistory().setQueryProperty(queryId,
Keys.QUERY_NUM_TASKS, String.valueOf(jobs));
- boolean hasReduce = hasReduceTasks(sem.getRootTasks());
-
- if (hasReduce) {
- console.printInfo("Number of reducers = "
- + conf.getIntVar(HiveConf.ConfVars.HADOOPNUMREDUCERS));
- console.printInfo("In order to change numer of reducers use:");
- console.printInfo(" set mapred.reduce.tasks = <number>");
- }
-
String jobname = Utilities.abbreviate(command, maxlen - 6);
int curJob = 0;
for (Task<? extends Serializable> rootTask : sem.getRootTasks()) {
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java?rev=738727&r1=738726&r2=738727&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java Thu Jan 29 02:46:41 2009
@@ -29,6 +29,7 @@
import org.apache.commons.logging.LogFactory;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.io.*;
@@ -48,7 +49,6 @@
public class ExecDriver extends Task<mapredWork> implements Serializable {
private static final long serialVersionUID = 1L;
- public static final long LOAD_PER_REDUCER = 1024 * 1024 * 1024;
transient protected JobConf job;
@@ -109,24 +109,6 @@
console = new LogHelper(LOG, isSilent);
}
- protected void fillInDefaults() {
- // this is a temporary hack to fix things that are not fixed in the compiler
- if (work.getNumReduceTasks() == null) {
- if (work.getReducer() == null) {
- LOG
- .warn("Number of reduce tasks not specified. Defaulting to 0 since there's no reduce operator");
- work.setNumReduceTasks(Integer.valueOf(0));
- } else {
- LOG
- .warn("Number of reduce tasks not specified. Defaulting to jobconf value of: "
- + job.getNumReduceTasks());
- work.setNumReduceTasks(job.getNumReduceTasks());
- }
- } else
- LOG.info("Number of reduce tasks determined at compile : "
- + work.getNumReduceTasks());
- }
-
/**
* A list of the currently running jobs spawned in this Hive instance that is
* used to kill all running jobs in the event of an unexpected shutdown -
@@ -223,34 +205,61 @@
return rj;
}
- private void inferNumReducers() throws Exception {
- FileSystem fs = FileSystem.get(job);
-
- if ((work.getReducer() != null) && (work.getInferNumReducers() == true)) {
- long inpSz = 0;
-
- // based on the input size - estimate the number of reducers
- Path[] inputPaths = FileInputFormat.getInputPaths(job);
-
- for (Path inputP : inputPaths) {
- if (fs.exists(inputP)) {
- FileStatus[] fStats = fs.listStatus(inputP);
- for (FileStatus fStat : fStats)
- inpSz += fStat.getLen();
- }
- }
-
- int newRed = (int) (inpSz / LOAD_PER_REDUCER) + 1;
- if (newRed < work.getNumReduceTasks().intValue()) {
+ /**
+ * Estimate the number of reducers needed for this job, based on job input,
+ * and configuration parameters.
+ * @return the number of reducers.
+ */
+ public int estimateNumberOfReducers(HiveConf hive, JobConf job, mapredWork work) throws IOException {
+ if (hive == null) {
+ hive = new HiveConf();
+ }
+ long bytesPerReducer = hive.getLongVar(HiveConf.ConfVars.BYTESPERREDUCER);
+ int maxReducers = hive.getIntVar(HiveConf.ConfVars.MAXREDUCERS);
+ long totalInputFileSize = getTotalInputFileSize(job, work);
+
+ LOG.info("BytesPerReducer=" + bytesPerReducer + " maxReducers=" + maxReducers
+ + " totalInputFileSize=" + totalInputFileSize);
+
+ int reducers = (int)((totalInputFileSize + bytesPerReducer - 1) / bytesPerReducer);
+ reducers = Math.max(1, reducers);
+ reducers = Math.min(maxReducers, reducers);
+ return reducers;
+ }
- LOG.warn("Number of reduce tasks inferred based on input size to : "
- + newRed);
- work.setNumReduceTasks(Integer.valueOf(newRed));
+ /**
+ * Set the number of reducers for the mapred work.
+ */
+ protected void setNumberOfReducers() throws IOException {
+ // this is a temporary hack to fix things that are not fixed in the compiler
+ Integer numReducersFromWork = work.getNumReduceTasks();
+
+ if(work.getReducer() == null) {
+ console.printInfo("Number of reduce tasks is set to 0 since there's no reduce operator");
+ work.setNumReduceTasks(Integer.valueOf(0));
+ } else {
+ if (numReducersFromWork >= 0) {
+ console.printInfo("Number of reduce tasks determined at compile time: " + work.getNumReduceTasks());
+ } else if (job.getNumReduceTasks() > 0) {
+ int reducers = job.getNumReduceTasks();
+ work.setNumReduceTasks(reducers);
+ console.printInfo("Number of reduce tasks not specified. Defaulting to jobconf value of: " + reducers);
+ } else {
+ int reducers = estimateNumberOfReducers(conf, job, work);
+ work.setNumReduceTasks(reducers);
+ console.printInfo("Number of reduce tasks not specified. Estimated from input data size: " + reducers);
}
+ console.printInfo("In order to change the average load for a reducer (in bytes):");
+ console.printInfo(" set " + HiveConf.ConfVars.BYTESPERREDUCER.varname + "=<number>");
+ console.printInfo("In order to limit the maximum number of reducers:");
+ console.printInfo(" set " + HiveConf.ConfVars.MAXREDUCERS.varname + "=<number>");
+ console.printInfo("In order to set a constant number of reducers:");
+ console.printInfo(" set " + HiveConf.ConfVars.HADOOPNUMREDUCERS + "=<number>");
}
}
+
/**
* Add new elements to the classpath
*
@@ -275,11 +284,39 @@
}
/**
+ * Calculate the total size of input files.
+ * @param job the hadoop job conf.
+ * @return the total size in bytes.
+ * @throws IOException
+ */
+ public long getTotalInputFileSize(JobConf job, mapredWork work) throws IOException {
+ long r = 0;
+ FileSystem fs = FileSystem.get(job);
+ // For each input path, calculate the total size.
+ for (String path: work.getPathToAliases().keySet()) {
+ try {
+ ContentSummary cs = fs.getContentSummary(new Path(path));
+ r += cs.getLength();
+ } catch (IOException e) {
+ LOG.info("Cannot get size of " + path + ". Safely ignored.");
+ }
+ }
+ return r;
+ }
+
+
+ /**
* Execute a query plan using Hadoop
*/
public int execute() {
- fillInDefaults();
+ try {
+ setNumberOfReducers();
+ } catch(IOException e) {
+ String statusMesg = "IOException occurred while acceesing HDFS to estimate the number of reducers.";
+ console.printError(statusMesg);
+ return 1;
+ }
String invalidReason = work.isInvalid();
if (invalidReason != null) {
@@ -344,20 +381,6 @@
return 0;
}
- inferNumReducers();
-
-
- if (SessionState.get() != null) {
- if (work.getReducer() != null) {
- SessionState.get().getHiveHistory().setTaskProperty(
- SessionState.get().getQueryId(), getId(),
- Keys.TASK_NUM_REDUCERS, String.valueOf(work.getNumReduceTasks()));
- } else {
- SessionState.get().getHiveHistory().setTaskProperty(
- SessionState.get().getQueryId(), getId(),
- Keys.TASK_NUM_REDUCERS, String.valueOf(0));
- }
- }
JobClient jc = new JobClient(job);
// make this client wait if job trcker is not behaving well.
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java?rev=738727&r1=738726&r2=738727&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java Thu Jan 29 02:46:41 2009
@@ -82,13 +82,7 @@
plan.setReducer(reducer);
reduceSinkDesc desc = (reduceSinkDesc)op.getConf();
- // The number of reducers may be specified in the plan in some cases, or may need to be inferred
- if (desc != null) {
- if (desc.getNumReducers() != -1)
- plan.setNumReduceTasks(new Integer(desc.getNumReducers()));
- else if (desc.getInferNumReducers() == true)
- plan.setInferNumReducers(true);
- }
+ plan.setNumReduceTasks(desc.getNumReducers());
List<Task<? extends Serializable>> rootTasks = opProcCtx.getRootTasks();
@@ -161,10 +155,7 @@
cplan.setReducer(reducer);
reduceSinkDesc desc = (reduceSinkDesc)op.getConf();
- if (desc.getNumReducers() != -1)
- cplan.setNumReduceTasks(new Integer(desc.getNumReducers()));
- else
- cplan.setInferNumReducers(desc.getInferNumReducers());
+ cplan.setNumReduceTasks(new Integer(desc.getNumReducers()));
HashMap<Operator<? extends Serializable>, Task<? extends Serializable>> opTaskMap = opProcCtx.getOpTaskMap();
opTaskMap.put(reducer, redTask);
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=738727&r1=738726&r2=738727&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Thu Jan 29 02:46:41 2009
@@ -1552,12 +1552,12 @@
String dest, Operator inputOperatorInfo)
throws SemanticException {
- return genGroupByPlanReduceSinkOperator(parseInfo, dest, inputOperatorInfo, -1, false);
+ return genGroupByPlanReduceSinkOperator(parseInfo, dest, inputOperatorInfo, -1);
}
@SuppressWarnings("nls")
private Operator genGroupByPlanReduceSinkOperator(QBParseInfo parseInfo,
- String dest, Operator inputOperatorInfo, int numReducers, boolean inferNumReducers)
+ String dest, Operator inputOperatorInfo, int numReducers)
throws SemanticException {
RowResolver reduceSinkInputRowResolver = opParseCtx.get(inputOperatorInfo).getRR();
RowResolver reduceSinkOutputRowResolver = new RowResolver();
@@ -1616,7 +1616,7 @@
return putOpInsertMap(
OperatorFactory.getAndMakeChild(
PlanUtils.getReduceSinkDesc(reduceKeys, reduceValues, -1,
- (parseInfo.getDistinctFuncExprForClause(dest) == null ? -1 : Integer.MAX_VALUE), numReducers, inferNumReducers),
+ (parseInfo.getDistinctFuncExprForClause(dest) == null ? -1 : Integer.MAX_VALUE), numReducers),
new RowSchema(reduceSinkOutputRowResolver.getColumnInfos()),
inputOperatorInfo),
reduceSinkOutputRowResolver);
@@ -1695,7 +1695,7 @@
return putOpInsertMap(
OperatorFactory.getAndMakeChild(PlanUtils.getReduceSinkDesc(reduceKeys, reduceValues, -1, numPartitionFields,
- -1, false),
+ -1),
new RowSchema(reduceSinkOutputRowResolver.getColumnInfos()),
inputOperatorInfo),
reduceSinkOutputRowResolver
@@ -1752,7 +1752,7 @@
return putOpInsertMap(
OperatorFactory.getAndMakeChild(PlanUtils.getReduceSinkDesc(reduceKeys, reduceValues, -1,
- numPartitionFields, -1, true),
+ numPartitionFields, -1),
new RowSchema(reduceSinkOutputRowResolver2.getColumnInfos()),
groupByOperatorInfo),
reduceSinkOutputRowResolver2
@@ -1959,7 +1959,7 @@
else {
// ////// Generate ReduceSink Operator
Operator reduceSinkOperatorInfo =
- genGroupByPlanReduceSinkOperator(parseInfo, dest, groupByOperatorInfo, 1, false);
+ genGroupByPlanReduceSinkOperator(parseInfo, dest, groupByOperatorInfo, 1);
return genGroupByPlanGroupByOperator2MR(parseInfo, dest, reduceSinkOperatorInfo, groupByDesc.Mode.FINAL);
}
@@ -2243,7 +2243,7 @@
Operator interim = putOpInsertMap(
OperatorFactory.getAndMakeChild(
PlanUtils.getReduceSinkDesc(sortCols, valueCols, -1, partitionCols, order.toString(),
- numReducers, false),
+ numReducers),
new RowSchema(inputRR.getColumnInfos()),
input), inputRR);
@@ -2353,7 +2353,7 @@
return putOpInsertMap(
OperatorFactory.getAndMakeChild(
- PlanUtils.getReduceSinkDesc(reduceKeys, reduceValues, joinTree.getNextTag(), reduceKeys.size(), -1, false),
+ PlanUtils.getReduceSinkDesc(reduceKeys, reduceValues, joinTree.getNextTag(), reduceKeys.size(), -1),
new RowSchema(outputRS.getColumnInfos()),
child), outputRS);
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java?rev=738727&r1=738726&r2=738727&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java Thu Jan 29 02:46:41 2009
@@ -179,8 +179,8 @@
* @param valueCols The columns to be stored in the value
* @param tag The tag for this reducesink
* @param partitionCols The columns for partitioning.
- * @param numReducers The number of reducers.
- * @param inferNumReducers whether we should try to infer the number of reducers.
+ * @param numReducers The number of reducers, set to -1 for automatic inference
+ * based on input data size.
* @return The reduceSinkDesc object.
*/
public static reduceSinkDesc getReduceSinkDesc(ArrayList<exprNodeDesc> keyCols,
@@ -188,9 +188,9 @@
int tag,
ArrayList<exprNodeDesc> partitionCols,
String order,
- int numReducers, boolean inferNumReducers) {
+ int numReducers) {
- return new reduceSinkDesc(keyCols, valueCols, tag, partitionCols, numReducers, inferNumReducers,
+ return new reduceSinkDesc(keyCols, valueCols, tag, partitionCols, numReducers,
getBinarySortableTableDesc(getFieldSchemasFromColumnList(keyCols, "reducesinkkey"), order),
getBinaryTableDesc(getFieldSchemasFromColumnList(valueCols, "reducesinkvalue")));
}
@@ -202,15 +202,15 @@
* @param tag The tag for this reducesink
* @param numPartitionFields The first numPartitionFields of keyCols will be partition columns.
* If numPartitionFields=-1, then partition randomly.
- * @param numReducers The number of reducers.
- * @param inferNumReducers whether we should try to infer the number of reducers.
+ * @param numReducers The number of reducers, set to -1 for automatic inference
+ * based on input data size.
* @return The reduceSinkDesc object.
*/
public static reduceSinkDesc getReduceSinkDesc(ArrayList<exprNodeDesc> keyCols,
ArrayList<exprNodeDesc> valueCols,
int tag,
int numPartitionFields,
- int numReducers, boolean inferNumReducers) {
+ int numReducers) {
ArrayList<exprNodeDesc> partitionCols = null;
if (numPartitionFields >= keyCols.size()) {
@@ -231,7 +231,7 @@
order.append("+");
}
return getReduceSinkDesc(keyCols, valueCols, tag, partitionCols, order.toString(),
- numReducers, inferNumReducers);
+ numReducers);
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/mapredWork.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/mapredWork.java?rev=738727&r1=738726&r2=738727&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/mapredWork.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/mapredWork.java Thu Jan 29 02:46:41 2009
@@ -49,7 +49,6 @@
private Integer numReduceTasks;
private boolean needsTagging;
- private boolean inferNumReducers;
public mapredWork() { }
public mapredWork(
@@ -122,7 +121,15 @@
this.reducer = reducer;
}
- @explain(displayName="# Reducers")
+ /**
+ * If the number of reducers is -1, the runtime will automatically
+ * figure it out by input data size.
+ *
+ * The number of reducers will be a positive number only in case the
+ * target table is bucketed into N buckets (through CREATE TABLE).
+ * This feature is not supported yet, so the number of reducers will
+ * always be -1 for now.
+ */
public Integer getNumReduceTasks() {
return this.numReduceTasks;
}
@@ -199,12 +206,4 @@
this.needsTagging = needsTagging;
}
- public boolean getInferNumReducers() {
- return this.inferNumReducers;
- }
-
- public void setInferNumReducers(boolean inferNumReducers) {
- this.inferNumReducers = inferNumReducers;
- }
-
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/reduceSinkDesc.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/reduceSinkDesc.java?rev=738727&r1=738726&r2=738727&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/reduceSinkDesc.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/reduceSinkDesc.java Thu Jan 29 02:46:41 2009
@@ -52,7 +52,6 @@
*/
private java.util.ArrayList<exprNodeDesc> partitionCols;
- private boolean inferNumReducers;
private int numReducers;
public reduceSinkDesc() { }
@@ -63,14 +62,12 @@
int tag,
java.util.ArrayList<exprNodeDesc> partitionCols,
int numReducers,
- boolean inferNumReducers,
final tableDesc keySerializeInfo,
final tableDesc valueSerializeInfo) {
this.keyCols = keyCols;
this.valueCols = valueCols;
this.tag = tag;
this.numReducers = numReducers;
- this.inferNumReducers = inferNumReducers;
this.partitionCols = partitionCols;
this.keySerializeInfo = keySerializeInfo;
this.valueSerializeInfo = valueSerializeInfo;
@@ -110,13 +107,12 @@
this.tag = tag;
}
- public boolean getInferNumReducers() {
- return this.inferNumReducers;
- }
- public void setInferNumReducers(boolean inferNumReducers) {
- this.inferNumReducers = inferNumReducers;
- }
-
+ /**
+ * Returns the number of reducers for the map-reduce job.
+ * -1 means to decide the number of reducers at runtime. This enables Hive to estimate
+ * the number of reducers based on the map-reduce input data size, which is only
+ * available right before we start the map-reduce job.
+ */
public int getNumReducers() {
return this.numReducers;
}
Modified: hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java?rev=738727&r1=738726&r2=738727&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java (original)
+++ hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java Thu Jan 29 02:46:41 2009
@@ -219,7 +219,7 @@
Operator<reduceSinkDesc> op1 = OperatorFactory.get
(PlanUtils.getReduceSinkDesc
(Utilities.makeList(new exprNodeColumnDesc(String.class, "key")),
- Utilities.makeList(new exprNodeColumnDesc(String.class, "value")), -1, 1, -1, false));
+ Utilities.makeList(new exprNodeColumnDesc(String.class, "value")), -1, 1, -1));
Utilities.addMapWork(mr, src, "a", op1);
mr.setKeyDesc(op1.getConf().getKeySerializeInfo());
@@ -245,7 +245,7 @@
(PlanUtils.getReduceSinkDesc
(Utilities.makeList(new exprNodeColumnDesc(String.class, "key")),
Utilities.makeList(new exprNodeColumnDesc(String.class, "key"),
- new exprNodeColumnDesc(String.class, "value")), -1, 1, -1, false));
+ new exprNodeColumnDesc(String.class, "value")), -1, 1, -1));
Utilities.addMapWork(mr, src, "a", op1);
mr.setKeyDesc(op1.getConf().getKeySerializeInfo());
@@ -278,7 +278,7 @@
(PlanUtils.getReduceSinkDesc
(Utilities.makeList(new exprNodeColumnDesc(String.class, "key")),
Utilities.makeList
- (new exprNodeColumnDesc(String.class, "value")), Byte.valueOf((byte)0), 1, -1, false));
+ (new exprNodeColumnDesc(String.class, "value")), Byte.valueOf((byte)0), 1, -1));
Utilities.addMapWork(mr, src, "a", op1);
mr.setKeyDesc(op1.getConf().getKeySerializeInfo());
@@ -289,7 +289,7 @@
(Utilities.makeList(new exprNodeColumnDesc(String.class, "key")),
Utilities.makeList(new exprNodeColumnDesc(String.class, "key")),
Byte.valueOf((byte)1),
- Integer.MAX_VALUE, -1, false));
+ Integer.MAX_VALUE, -1));
Utilities.addMapWork(mr, src2, "b", op2);
mr.getTagToValueDesc().add(op2.getConf().getValueSerializeInfo());
@@ -324,7 +324,7 @@
(Utilities.makeList(new exprNodeColumnDesc(String.class, "tkey")),
Utilities.makeList(new exprNodeColumnDesc(String.class, "tkey"),
new exprNodeColumnDesc(String.class, "tvalue")),
- -1, 1, -1, false));
+ -1, 1, -1));
Operator<scriptDesc> op0 = OperatorFactory.get
(new scriptDesc("/bin/cat",
@@ -362,7 +362,7 @@
(Utilities.makeList(new exprNodeColumnDesc(String.class, "0")),
Utilities.makeList(new exprNodeColumnDesc(String.class, "0"),
new exprNodeColumnDesc(String.class, "1")),
- -1, 1, -1, false));
+ -1, 1, -1));
Operator<selectDesc> op4 = OperatorFactory.get(new selectDesc(
Utilities.makeList(new exprNodeColumnDesc(String.class, "key"),
@@ -394,7 +394,7 @@
Utilities.makeList(new exprNodeColumnDesc(String.class, "tkey")),
Utilities.makeList(new exprNodeColumnDesc(String.class, "tkey"),
new exprNodeColumnDesc(String.class, "tvalue")),
- -1, 1, -1, false));
+ -1, 1, -1));
Operator<scriptDesc> op0 = OperatorFactory.get
(new scriptDesc("\'/bin/cat\'",
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_limit.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_limit.q.out?rev=738727&r1=738726&r2=738727&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_limit.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby1_limit.q.out Thu Jan 29 02:46:41 2009
@@ -42,7 +42,7 @@
Stage: Stage-2
Map Reduce
Alias -> Map Operator Tree:
- /tmp/hive-zshao/618493432/49810635.10001
+ /data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/tmp/228651165/7195775.10001
Reduce Output Operator
key expressions:
expr: 0
@@ -80,7 +80,7 @@
Stage: Stage-3
Map Reduce
Alias -> Map Operator Tree:
- /tmp/hive-zshao/618493432/49810635.10002
+ /data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/tmp/228651165/7195775.10002
Reduce Output Operator
sort order:
tag: -1
@@ -89,7 +89,6 @@
type: string
expr: 1
type: double
- # Reducers: 1
Reduce Operator Tree:
Extract
Limit
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby4_map.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby4_map.q.out?rev=738727&r1=738726&r2=738727&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby4_map.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby4_map.q.out Thu Jan 29 02:46:41 2009
@@ -24,7 +24,6 @@
value expressions:
expr: 0
type: bigint
- # Reducers: 1
Reduce Operator Tree:
Group By Operator
aggregations:
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby5_map.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby5_map.q.out?rev=738727&r1=738726&r2=738727&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby5_map.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/groupby5_map.q.out Thu Jan 29 02:46:41 2009
@@ -27,7 +27,6 @@
value expressions:
expr: 0
type: double
- # Reducers: 1
Reduce Operator Tree:
Group By Operator
aggregations:
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/input11_limit.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/input11_limit.q.out?rev=738727&r1=738726&r2=738727&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/input11_limit.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/input11_limit.q.out Thu Jan 29 02:46:41 2009
@@ -29,7 +29,6 @@
type: string
expr: 1
type: string
- # Reducers: 1
Reduce Operator Tree:
Extract
Limit
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/input14_limit.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/input14_limit.q.out?rev=738727&r1=738726&r2=738727&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/input14_limit.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/input14_limit.q.out Thu Jan 29 02:46:41 2009
@@ -49,7 +49,7 @@
Stage: Stage-2
Map Reduce
Alias -> Map Operator Tree:
- /data/users/athusoo/apacheprojects/hive_local_ws3/build/ql/tmp/456652865/144848153.10001
+ /data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/tmp/360702650/134814302.10001
Reduce Output Operator
key expressions:
expr: 0
@@ -64,7 +64,6 @@
type: string
expr: 1
type: string
- # Reducers: 1
Reduce Operator Tree:
Extract
Limit
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/input1_limit.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/input1_limit.q.out?rev=738727&r1=738726&r2=738727&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/input1_limit.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/input1_limit.q.out Thu Jan 29 02:46:41 2009
@@ -48,7 +48,6 @@
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.mapred.SequenceFileOutputFormat
name: binary_table
- # Reducers: 1
Reduce Operator Tree:
Extract
Limit
@@ -85,7 +84,7 @@
Stage: Stage-2
Map Reduce
Alias -> Map Operator Tree:
- /data/users/athusoo/apacheprojects/hive_local_ws3/build/ql/tmp/2110365113/704872924.10002
+ /data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/tmp/1425158066/374907939.10002
Reduce Output Operator
sort order:
tag: -1
@@ -94,7 +93,6 @@
type: string
expr: 1
type: string
- # Reducers: 1
Reduce Operator Tree:
Extract
Limit
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/input3_limit.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/input3_limit.q.out?rev=738727&r1=738726&r2=738727&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/input3_limit.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/input3_limit.q.out Thu Jan 29 02:46:41 2009
@@ -25,7 +25,6 @@
type: string
expr: 1
type: string
- # Reducers: 1
Reduce Operator Tree:
Extract
Limit
Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby1.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby1.q.xml?rev=738727&r1=738726&r2=738727&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby1.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby1.q.xml Thu Jan 29 02:46:41 2009
@@ -35,7 +35,7 @@
<boolean>true</boolean>
</void>
<void property="sourceDir">
- <string>/tmp/hive-njain/1734876980/99648103.10000.insclause-0</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/ql/../build/ql/tmp/361270251/169850988.10000.insclause-0</string>
</void>
<void property="table">
<object id="tableDesc0" class="org.apache.hadoop.hive.ql.plan.tableDesc">
@@ -84,7 +84,7 @@
</void>
<void method="put">
<string>location</string>
- <string>file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest1</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/test/data/warehouse/dest1</string>
</void>
</object>
</void>
@@ -118,13 +118,10 @@
<void property="aliasToWork">
<object class="java.util.HashMap">
<void method="put">
- <string>/tmp/hive-njain/1734876980/99648103.10001</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/tmp/361270251/169850988.10001</string>
<object id="ReduceSinkOperator0" class="org.apache.hadoop.hive.ql.exec.ReduceSinkOperator">
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.reduceSinkDesc">
- <void property="inferNumReducers">
- <boolean>true</boolean>
- </void>
<void property="keyCols">
<object id="ArrayList0" class="java.util.ArrayList">
<void method="add">
@@ -245,7 +242,7 @@
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc">
<void property="dirName">
- <string>/tmp/hive-njain/1734876980/99648103.10001</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/tmp/361270251/169850988.10001</string>
</void>
<void property="tableInfo">
<object id="tableDesc3" class="org.apache.hadoop.hive.ql.plan.tableDesc">
@@ -671,19 +668,19 @@
</void>
</object>
</void>
- <void property="inferNumReducers">
- <boolean>true</boolean>
- </void>
<void property="keyDesc">
<object idref="tableDesc1"/>
</void>
+ <void property="numReduceTasks">
+ <int>-1</int>
+ </void>
<void property="pathToAliases">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>/tmp/hive-njain/1734876980/99648103.10001</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/tmp/361270251/169850988.10001</string>
<object class="java.util.ArrayList">
<void method="add">
- <string>/tmp/hive-njain/1734876980/99648103.10001</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/tmp/361270251/169850988.10001</string>
</void>
</object>
</void>
@@ -692,7 +689,7 @@
<void property="pathToPartitionInfo">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>/tmp/hive-njain/1734876980/99648103.10001</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/tmp/361270251/169850988.10001</string>
<object class="org.apache.hadoop.hive.ql.plan.partitionDesc">
<void property="tableDesc">
<object idref="tableDesc3"/>
@@ -714,7 +711,7 @@
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc">
<void property="dirName">
- <string>/tmp/hive-njain/1734876980/99648103.10000.insclause-0</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/ql/../build/ql/tmp/361270251/169850988.10000.insclause-0</string>
</void>
<void property="tableInfo">
<object idref="tableDesc0"/>
@@ -920,10 +917,13 @@
<void property="keyDesc">
<object idref="tableDesc4"/>
</void>
+ <void property="numReduceTasks">
+ <int>-1</int>
+ </void>
<void property="pathToAliases">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/test/data/warehouse/src</string>
<object class="java.util.ArrayList">
<void method="add">
<string>src</string>
@@ -935,7 +935,7 @@
<void property="pathToPartitionInfo">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/test/data/warehouse/src</string>
<object class="org.apache.hadoop.hive.ql.plan.partitionDesc">
<void property="partSpec">
<object class="java.util.LinkedHashMap"/>
@@ -987,7 +987,7 @@
</void>
<void method="put">
<string>location</string>
- <string>file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/test/data/warehouse/src</string>
</void>
</object>
</void>
Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby2.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby2.q.xml?rev=738727&r1=738726&r2=738727&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby2.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby2.q.xml Thu Jan 29 02:46:41 2009
@@ -20,13 +20,10 @@
<void property="aliasToWork">
<object class="java.util.HashMap">
<void method="put">
- <string>/tmp/hive-njain/568944209/29322851.10002</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/tmp/187708922/156247810.10002</string>
<object id="ReduceSinkOperator0" class="org.apache.hadoop.hive.ql.exec.ReduceSinkOperator">
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.reduceSinkDesc">
- <void property="inferNumReducers">
- <boolean>true</boolean>
- </void>
<void property="keyCols">
<object id="ArrayList0" class="java.util.ArrayList">
<void method="add">
@@ -161,7 +158,7 @@
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc">
<void property="dirName">
- <string>/tmp/hive-njain/568944209/29322851.10002</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/tmp/187708922/156247810.10002</string>
</void>
<void property="tableInfo">
<object id="tableDesc2" class="org.apache.hadoop.hive.ql.plan.tableDesc">
@@ -660,19 +657,19 @@
</void>
</object>
</void>
- <void property="inferNumReducers">
- <boolean>true</boolean>
- </void>
<void property="keyDesc">
<object idref="tableDesc0"/>
</void>
+ <void property="numReduceTasks">
+ <int>-1</int>
+ </void>
<void property="pathToAliases">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>/tmp/hive-njain/568944209/29322851.10002</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/tmp/187708922/156247810.10002</string>
<object class="java.util.ArrayList">
<void method="add">
- <string>/tmp/hive-njain/568944209/29322851.10002</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/tmp/187708922/156247810.10002</string>
</void>
</object>
</void>
@@ -681,7 +678,7 @@
<void property="pathToPartitionInfo">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>/tmp/hive-njain/568944209/29322851.10002</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/tmp/187708922/156247810.10002</string>
<object class="org.apache.hadoop.hive.ql.plan.partitionDesc">
<void property="tableDesc">
<object idref="tableDesc2"/>
@@ -703,7 +700,7 @@
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc">
<void property="dirName">
- <string>/tmp/hive-njain/568944209/29322851.10001.insclause-0</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/ql/../build/ql/tmp/187708922/156247810.10001.insclause-0</string>
</void>
<void property="tableInfo">
<object class="org.apache.hadoop.hive.ql.plan.tableDesc">
@@ -1043,10 +1040,13 @@
<void property="keyDesc">
<object idref="tableDesc3"/>
</void>
+ <void property="numReduceTasks">
+ <int>-1</int>
+ </void>
<void property="pathToAliases">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/test/data/warehouse/src</string>
<object class="java.util.ArrayList">
<void method="add">
<string>src</string>
@@ -1058,7 +1058,7 @@
<void property="pathToPartitionInfo">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/test/data/warehouse/src</string>
<object class="org.apache.hadoop.hive.ql.plan.partitionDesc">
<void property="partSpec">
<object class="java.util.LinkedHashMap"/>
@@ -1110,7 +1110,7 @@
</void>
<void method="put">
<string>location</string>
- <string>file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/test/data/warehouse/src</string>
</void>
</object>
</void>
Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby3.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby3.q.xml?rev=738727&r1=738726&r2=738727&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby3.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby3.q.xml Thu Jan 29 02:46:41 2009
@@ -20,13 +20,10 @@
<void property="aliasToWork">
<object class="java.util.HashMap">
<void method="put">
- <string>/data/users/athusoo/commits/hive_trunk_ws3/build/ql/tmp/221218874/81025690.10002</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/tmp/6568832/173702865.10002</string>
<object id="ReduceSinkOperator0" class="org.apache.hadoop.hive.ql.exec.ReduceSinkOperator">
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.reduceSinkDesc">
- <void property="inferNumReducers">
- <boolean>true</boolean>
- </void>
<void property="keyCols">
<object id="ArrayList0" class="java.util.ArrayList"/>
</void>
@@ -176,7 +173,7 @@
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc">
<void property="dirName">
- <string>/data/users/athusoo/commits/hive_trunk_ws3/build/ql/tmp/221218874/81025690.10002</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/tmp/6568832/173702865.10002</string>
</void>
<void property="tableInfo">
<object id="tableDesc2" class="org.apache.hadoop.hive.ql.plan.tableDesc">
@@ -796,19 +793,19 @@
</void>
</object>
</void>
- <void property="inferNumReducers">
- <boolean>true</boolean>
- </void>
<void property="keyDesc">
<object idref="tableDesc0"/>
</void>
+ <void property="numReduceTasks">
+ <int>-1</int>
+ </void>
<void property="pathToAliases">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>/data/users/athusoo/commits/hive_trunk_ws3/build/ql/tmp/221218874/81025690.10002</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/tmp/6568832/173702865.10002</string>
<object class="java.util.ArrayList">
<void method="add">
- <string>/data/users/athusoo/commits/hive_trunk_ws3/build/ql/tmp/221218874/81025690.10002</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/tmp/6568832/173702865.10002</string>
</void>
</object>
</void>
@@ -817,7 +814,7 @@
<void property="pathToPartitionInfo">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>/data/users/athusoo/commits/hive_trunk_ws3/build/ql/tmp/221218874/81025690.10002</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/tmp/6568832/173702865.10002</string>
<object class="org.apache.hadoop.hive.ql.plan.partitionDesc">
<void property="tableDesc">
<object idref="tableDesc2"/>
@@ -839,7 +836,7 @@
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc">
<void property="dirName">
- <string>/data/users/athusoo/commits/hive_trunk_ws3/ql/../build/ql/tmp/482395674.10001.insclause-0</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/ql/../build/ql/tmp/1072875330.10001.insclause-0</string>
</void>
<void property="tableInfo">
<object class="org.apache.hadoop.hive.ql.plan.tableDesc">
@@ -1278,10 +1275,13 @@
<void property="keyDesc">
<object idref="tableDesc3"/>
</void>
+ <void property="numReduceTasks">
+ <int>-1</int>
+ </void>
<void property="pathToAliases">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/data/users/athusoo/commits/hive_trunk_ws3/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/test/data/warehouse/src</string>
<object class="java.util.ArrayList">
<void method="add">
<string>src</string>
@@ -1293,7 +1293,7 @@
<void property="pathToPartitionInfo">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/data/users/athusoo/commits/hive_trunk_ws3/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/test/data/warehouse/src</string>
<object class="org.apache.hadoop.hive.ql.plan.partitionDesc">
<void property="partSpec">
<object class="java.util.LinkedHashMap"/>
@@ -1345,7 +1345,7 @@
</void>
<void method="put">
<string>location</string>
- <string>file:/data/users/athusoo/commits/hive_trunk_ws3/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/test/data/warehouse/src</string>
</void>
</object>
</void>
Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby4.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby4.q.xml?rev=738727&r1=738726&r2=738727&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby4.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby4.q.xml Thu Jan 29 02:46:41 2009
@@ -20,13 +20,10 @@
<void property="aliasToWork">
<object class="java.util.HashMap">
<void method="put">
- <string>/tmp/hive-njain/270959803/34104799.10002</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/tmp/42922291/496011429.10002</string>
<object id="ReduceSinkOperator0" class="org.apache.hadoop.hive.ql.exec.ReduceSinkOperator">
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.reduceSinkDesc">
- <void property="inferNumReducers">
- <boolean>true</boolean>
- </void>
<void property="keyCols">
<object id="ArrayList0" class="java.util.ArrayList">
<void method="add">
@@ -132,7 +129,7 @@
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc">
<void property="dirName">
- <string>/tmp/hive-njain/270959803/34104799.10002</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/tmp/42922291/496011429.10002</string>
</void>
<void property="tableInfo">
<object id="tableDesc2" class="org.apache.hadoop.hive.ql.plan.tableDesc">
@@ -539,19 +536,19 @@
</void>
</object>
</void>
- <void property="inferNumReducers">
- <boolean>true</boolean>
- </void>
<void property="keyDesc">
<object idref="tableDesc0"/>
</void>
+ <void property="numReduceTasks">
+ <int>-1</int>
+ </void>
<void property="pathToAliases">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>/tmp/hive-njain/270959803/34104799.10002</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/tmp/42922291/496011429.10002</string>
<object class="java.util.ArrayList">
<void method="add">
- <string>/tmp/hive-njain/270959803/34104799.10002</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/tmp/42922291/496011429.10002</string>
</void>
</object>
</void>
@@ -560,7 +557,7 @@
<void property="pathToPartitionInfo">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>/tmp/hive-njain/270959803/34104799.10002</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/tmp/42922291/496011429.10002</string>
<object class="org.apache.hadoop.hive.ql.plan.partitionDesc">
<void property="tableDesc">
<object idref="tableDesc2"/>
@@ -582,7 +579,7 @@
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc">
<void property="dirName">
- <string>/tmp/hive-njain/648245175.10001.insclause-0</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/ql/../build/ql/tmp/731171602.10001.insclause-0</string>
</void>
<void property="tableInfo">
<object class="org.apache.hadoop.hive.ql.plan.tableDesc">
@@ -758,10 +755,13 @@
<void property="keyDesc">
<object idref="tableDesc3"/>
</void>
+ <void property="numReduceTasks">
+ <int>-1</int>
+ </void>
<void property="pathToAliases">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/test/data/warehouse/src</string>
<object class="java.util.ArrayList">
<void method="add">
<string>src</string>
@@ -773,7 +773,7 @@
<void property="pathToPartitionInfo">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/test/data/warehouse/src</string>
<object class="org.apache.hadoop.hive.ql.plan.partitionDesc">
<void property="partSpec">
<object class="java.util.LinkedHashMap"/>
@@ -825,7 +825,7 @@
</void>
<void method="put">
<string>location</string>
- <string>file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/test/data/warehouse/src</string>
</void>
</object>
</void>
Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby5.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby5.q.xml?rev=738727&r1=738726&r2=738727&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby5.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby5.q.xml Thu Jan 29 02:46:41 2009
@@ -20,13 +20,10 @@
<void property="aliasToWork">
<object class="java.util.HashMap">
<void method="put">
- <string>/data/users/pchakka/workspace/oshive/build/ql/tmp/1685965608/18485413.10002</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/tmp/475879510/9101030.10002</string>
<object id="ReduceSinkOperator0" class="org.apache.hadoop.hive.ql.exec.ReduceSinkOperator">
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.reduceSinkDesc">
- <void property="inferNumReducers">
- <boolean>true</boolean>
- </void>
<void property="keyCols">
<object id="ArrayList0" class="java.util.ArrayList">
<void method="add">
@@ -147,7 +144,7 @@
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc">
<void property="dirName">
- <string>/data/users/pchakka/workspace/oshive/build/ql/tmp/1685965608/18485413.10002</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/tmp/475879510/9101030.10002</string>
</void>
<void property="tableInfo">
<object id="tableDesc2" class="org.apache.hadoop.hive.ql.plan.tableDesc">
@@ -573,19 +570,19 @@
</void>
</object>
</void>
- <void property="inferNumReducers">
- <boolean>true</boolean>
- </void>
<void property="keyDesc">
<object idref="tableDesc0"/>
</void>
+ <void property="numReduceTasks">
+ <int>-1</int>
+ </void>
<void property="pathToAliases">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>/data/users/pchakka/workspace/oshive/build/ql/tmp/1685965608/18485413.10002</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/tmp/475879510/9101030.10002</string>
<object class="java.util.ArrayList">
<void method="add">
- <string>/data/users/pchakka/workspace/oshive/build/ql/tmp/1685965608/18485413.10002</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/tmp/475879510/9101030.10002</string>
</void>
</object>
</void>
@@ -594,7 +591,7 @@
<void property="pathToPartitionInfo">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>/data/users/pchakka/workspace/oshive/build/ql/tmp/1685965608/18485413.10002</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/tmp/475879510/9101030.10002</string>
<object class="org.apache.hadoop.hive.ql.plan.partitionDesc">
<void property="tableDesc">
<object idref="tableDesc2"/>
@@ -616,7 +613,7 @@
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc">
<void property="dirName">
- <string>/data/users/pchakka/workspace/oshive/ql/../build/ql/tmp/1685965608/18485413.10001.insclause-0</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/ql/../build/ql/tmp/475879510/9101030.10001.insclause-0</string>
</void>
<void property="tableInfo">
<object class="org.apache.hadoop.hive.ql.plan.tableDesc">
@@ -844,10 +841,13 @@
<void property="keyDesc">
<object idref="tableDesc3"/>
</void>
+ <void property="numReduceTasks">
+ <int>-1</int>
+ </void>
<void property="pathToAliases">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/data/users/pchakka/workspace/oshive/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/test/data/warehouse/src</string>
<object class="java.util.ArrayList">
<void method="add">
<string>src</string>
@@ -859,7 +859,7 @@
<void property="pathToPartitionInfo">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/data/users/pchakka/workspace/oshive/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/test/data/warehouse/src</string>
<object class="org.apache.hadoop.hive.ql.plan.partitionDesc">
<void property="partSpec">
<object class="java.util.LinkedHashMap"/>
@@ -911,7 +911,7 @@
</void>
<void method="put">
<string>location</string>
- <string>file:/data/users/pchakka/workspace/oshive/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/test/data/warehouse/src</string>
</void>
</object>
</void>
Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby6.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby6.q.xml?rev=738727&r1=738726&r2=738727&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby6.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/groupby6.q.xml Thu Jan 29 02:46:41 2009
@@ -20,13 +20,10 @@
<void property="aliasToWork">
<object class="java.util.HashMap">
<void method="put">
- <string>/tmp/hive-njain/1062373643/636855904.10002</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/tmp/593011474/260610884.10002</string>
<object id="ReduceSinkOperator0" class="org.apache.hadoop.hive.ql.exec.ReduceSinkOperator">
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.reduceSinkDesc">
- <void property="inferNumReducers">
- <boolean>true</boolean>
- </void>
<void property="keyCols">
<object id="ArrayList0" class="java.util.ArrayList">
<void method="add">
@@ -132,7 +129,7 @@
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc">
<void property="dirName">
- <string>/tmp/hive-njain/1062373643/636855904.10002</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/tmp/593011474/260610884.10002</string>
</void>
<void property="tableInfo">
<object id="tableDesc2" class="org.apache.hadoop.hive.ql.plan.tableDesc">
@@ -539,19 +536,19 @@
</void>
</object>
</void>
- <void property="inferNumReducers">
- <boolean>true</boolean>
- </void>
<void property="keyDesc">
<object idref="tableDesc0"/>
</void>
+ <void property="numReduceTasks">
+ <int>-1</int>
+ </void>
<void property="pathToAliases">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>/tmp/hive-njain/1062373643/636855904.10002</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/tmp/593011474/260610884.10002</string>
<object class="java.util.ArrayList">
<void method="add">
- <string>/tmp/hive-njain/1062373643/636855904.10002</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/tmp/593011474/260610884.10002</string>
</void>
</object>
</void>
@@ -560,7 +557,7 @@
<void property="pathToPartitionInfo">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>/tmp/hive-njain/1062373643/636855904.10002</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/tmp/593011474/260610884.10002</string>
<object class="org.apache.hadoop.hive.ql.plan.partitionDesc">
<void property="tableDesc">
<object idref="tableDesc2"/>
@@ -582,7 +579,7 @@
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc">
<void property="dirName">
- <string>/tmp/hive-njain/786081691.10001.insclause-0</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/ql/../build/ql/tmp/14682144.10001.insclause-0</string>
</void>
<void property="tableInfo">
<object class="org.apache.hadoop.hive.ql.plan.tableDesc">
@@ -758,10 +755,13 @@
<void property="keyDesc">
<object idref="tableDesc3"/>
</void>
+ <void property="numReduceTasks">
+ <int>-1</int>
+ </void>
<void property="pathToAliases">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/test/data/warehouse/src</string>
<object class="java.util.ArrayList">
<void method="add">
<string>src</string>
@@ -773,7 +773,7 @@
<void property="pathToPartitionInfo">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/test/data/warehouse/src</string>
<object class="org.apache.hadoop.hive.ql.plan.partitionDesc">
<void property="partSpec">
<object class="java.util.LinkedHashMap"/>
@@ -825,7 +825,7 @@
</void>
<void method="put">
<string>location</string>
- <string>file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/test/data/warehouse/src</string>
</void>
</object>
</void>
Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/input20.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/input20.q.xml?rev=738727&r1=738726&r2=738727&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/input20.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/input20.q.xml Thu Jan 29 02:46:41 2009
@@ -600,10 +600,13 @@
<void property="keyDesc">
<object idref="tableDesc0"/>
</void>
+ <void property="numReduceTasks">
+ <int>-1</int>
+ </void>
<void property="pathToAliases">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/home/njain/workspace/fbhadoop/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/test/data/warehouse/src</string>
<object class="java.util.ArrayList">
<void method="add">
<string>tmap:src</string>
@@ -615,7 +618,7 @@
<void property="pathToPartitionInfo">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/home/njain/workspace/fbhadoop/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/test/data/warehouse/src</string>
<object class="org.apache.hadoop.hive.ql.plan.partitionDesc">
<void property="partSpec">
<object class="java.util.LinkedHashMap"/>
@@ -667,7 +670,7 @@
</void>
<void method="put">
<string>location</string>
- <string>file:/home/njain/workspace/fbhadoop/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/test/data/warehouse/src</string>
</void>
</object>
</void>
@@ -697,7 +700,7 @@
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc">
<void property="dirName">
- <string>/tmp/hive-njain/307350069.10001.insclause-0</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/ql/../build/ql/tmp/388201960.10001.insclause-0</string>
</void>
<void property="tableInfo">
<object class="org.apache.hadoop.hive.ql.plan.tableDesc">
Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/input4.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/input4.q.xml?rev=738727&r1=738726&r2=738727&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/input4.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/input4.q.xml Thu Jan 29 02:46:41 2009
@@ -31,7 +31,7 @@
<boolean>true</boolean>
</void>
<void property="sourceDir">
- <string>/data/users/athusoo/apacheprojects/hive_local_ws3/ql/../build/ql/tmp/692060421/96243067.10000.insclause-0</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/ql/../build/ql/tmp/117678603/347916488.10000.insclause-0</string>
</void>
<void property="table">
<object id="tableDesc0" class="org.apache.hadoop.hive.ql.plan.tableDesc">
@@ -80,7 +80,7 @@
</void>
<void method="put">
<string>location</string>
- <string>file:/data/users/athusoo/apacheprojects/hive_local_ws3/build/ql/test/data/warehouse/dest1</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/test/data/warehouse/dest1</string>
</void>
</object>
</void>
@@ -465,10 +465,13 @@
<void property="keyDesc">
<object idref="tableDesc1"/>
</void>
+ <void property="numReduceTasks">
+ <int>-1</int>
+ </void>
<void property="pathToAliases">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/data/users/athusoo/apacheprojects/hive_local_ws3/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/test/data/warehouse/src</string>
<object class="java.util.ArrayList">
<void method="add">
<string>tmap:src</string>
@@ -480,7 +483,7 @@
<void property="pathToPartitionInfo">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/data/users/athusoo/apacheprojects/hive_local_ws3/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/test/data/warehouse/src</string>
<object class="org.apache.hadoop.hive.ql.plan.partitionDesc">
<void property="partSpec">
<object class="java.util.LinkedHashMap"/>
@@ -532,7 +535,7 @@
</void>
<void method="put">
<string>location</string>
- <string>file:/data/users/athusoo/apacheprojects/hive_local_ws3/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/test/data/warehouse/src</string>
</void>
</object>
</void>
@@ -562,7 +565,7 @@
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc">
<void property="dirName">
- <string>/data/users/pchakka/workspace/oshive/ql/../build/ql/tmp/952811677/1477355585.10000.insclause-0</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/ql/../build/ql/tmp/117678603/347916488.10000.insclause-0</string>
</void>
<void property="tableInfo">
<object idref="tableDesc0"/>
Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/input5.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/input5.q.xml?rev=738727&r1=738726&r2=738727&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/input5.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/input5.q.xml Thu Jan 29 02:46:41 2009
@@ -31,7 +31,7 @@
<boolean>true</boolean>
</void>
<void property="sourceDir">
- <string>/tmp/hive-njain/113180245.10000.insclause-0</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/ql/../build/ql/tmp/768672036.10000.insclause-0</string>
</void>
<void property="table">
<object id="tableDesc0" class="org.apache.hadoop.hive.ql.plan.tableDesc">
@@ -80,7 +80,7 @@
</void>
<void method="put">
<string>location</string>
- <string>file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest1</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/test/data/warehouse/dest1</string>
</void>
</object>
</void>
@@ -599,10 +599,13 @@
<void property="keyDesc">
<object idref="tableDesc1"/>
</void>
+ <void property="numReduceTasks">
+ <int>-1</int>
+ </void>
<void property="pathToAliases">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src_thrift</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/test/data/warehouse/src_thrift</string>
<object class="java.util.ArrayList">
<void method="add">
<string>tmap:src_thrift</string>
@@ -614,7 +617,7 @@
<void property="pathToPartitionInfo">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src_thrift</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/test/data/warehouse/src_thrift</string>
<object class="org.apache.hadoop.hive.ql.plan.partitionDesc">
<void property="partSpec">
<object class="java.util.LinkedHashMap"/>
@@ -670,7 +673,7 @@
</void>
<void method="put">
<string>location</string>
- <string>file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src_thrift</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/test/data/warehouse/src_thrift</string>
</void>
</object>
</void>
@@ -696,7 +699,7 @@
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc">
<void property="dirName">
- <string>/tmp/hive-njain/113180245.10000.insclause-0</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/ql/../build/ql/tmp/768672036.10000.insclause-0</string>
</void>
<void property="tableInfo">
<object idref="tableDesc0"/>
Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/join1.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/join1.q.xml?rev=738727&r1=738726&r2=738727&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/join1.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/join1.q.xml Thu Jan 29 02:46:41 2009
@@ -31,7 +31,7 @@
<boolean>true</boolean>
</void>
<void property="sourceDir">
- <string>/tmp/hive-njain/605771515.10000.insclause-0</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/ql/../build/ql/tmp/707993971.10000.insclause-0</string>
</void>
<void property="table">
<object id="tableDesc0" class="org.apache.hadoop.hive.ql.plan.tableDesc">
@@ -80,7 +80,7 @@
</void>
<void method="put">
<string>location</string>
- <string>file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/dest1</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/test/data/warehouse/dest1</string>
</void>
</object>
</void>
@@ -524,10 +524,13 @@
<void property="needsTagging">
<boolean>true</boolean>
</void>
+ <void property="numReduceTasks">
+ <int>-1</int>
+ </void>
<void property="pathToAliases">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/test/data/warehouse/src</string>
<object class="java.util.ArrayList">
<void method="add">
<string>src2</string>
@@ -542,7 +545,7 @@
<void property="pathToPartitionInfo">
<object class="java.util.LinkedHashMap">
<void method="put">
- <string>file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/test/data/warehouse/src</string>
<object class="org.apache.hadoop.hive.ql.plan.partitionDesc">
<void property="partSpec">
<object class="java.util.LinkedHashMap"/>
@@ -594,7 +597,7 @@
</void>
<void method="put">
<string>location</string>
- <string>file:/home/njain/workspace/hadoophive/trunk/build/ql/test/data/warehouse/src</string>
+ <string>file:/data/users/zshao/sync/apache-trunk-HIVE-105/build/ql/test/data/warehouse/src</string>
</void>
</object>
</void>
@@ -620,7 +623,7 @@
<void property="conf">
<object class="org.apache.hadoop.hive.ql.plan.fileSinkDesc">
<void property="dirName">
- <string>/tmp/hive-njain/605771515.10000.insclause-0</string>
+ <string>/data/users/zshao/sync/apache-trunk-HIVE-105/ql/../build/ql/tmp/707993971.10000.insclause-0</string>
</void>
<void property="tableInfo">
<object idref="tableDesc0"/>