You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by nz...@apache.org on 2010/07/29 04:41:19 UTC
svn commit: r980297 [2/16] - in /hadoop/hive/trunk: ./
common/src/java/org/apache/hadoop/hive/common/
common/src/java/org/apache/hadoop/hive/conf/ conf/ data/conf/
metastore/src/test/org/apache/hadoop/hive/metastore/
ql/src/java/org/apache/hadoop/hive/...
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java?rev=980297&r1=980296&r2=980297&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java Thu Jul 29 02:41:14 2010
@@ -28,6 +28,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.QueryPlan;
import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.metadata.Hive;
@@ -123,18 +124,6 @@ public abstract class Task<T extends Ser
*/
protected abstract int execute(DriverContext driverContext);
- /**
- * Update the progress of the task within taskHandle and also dump the
- * progress information to the history file.
- *
- * @param taskHandle
- * task handle returned by execute
- * @throws IOException
- */
- public void progress(TaskHandle taskHandle) throws IOException {
- // do nothing by default
- }
-
// dummy method - FetchTask overwrites this
public boolean fetch(ArrayList<String> res) throws IOException {
assert false;
@@ -273,10 +262,6 @@ public abstract class Task<T extends Ser
return false;
}
- public void updateCounters(TaskHandle th) throws IOException {
- // default, do nothing
- }
-
public HashMap<String, Long> getCounters() {
return taskCounters;
}
@@ -291,4 +276,34 @@ public abstract class Task<T extends Ser
assert false;
return -1;
}
+
+ /**
+ * If this task uses any map-reduce intermediate data (either for reading
+ * or for writing), localize them (using the supplied Context). Map-Reduce
+ * intermediate directories are allocated using Context.getMRTmpFileURI()
+ * and can be localized using localizeMRTmpFileURI().
+ *
+ * This method is declared abstract to force any task code to explicitly
+ * deal with this aspect of execution.
+ *
+ * @param ctx context object with which to localize
+ */
+ abstract protected void localizeMRTmpFilesImpl(Context ctx);
+
+ /**
+ * Localize a task tree
+ * @param ctx context object with which to localize
+ */
+ public final void localizeMRTmpFiles(Context ctx) {
+ localizeMRTmpFilesImpl(ctx);
+
+ if (childTasks == null)
+ return;
+
+ for (Task<? extends Serializable> t: childTasks) {
+ t.localizeMRTmpFiles(ctx);
+ }
+ }
+
}
+
\ No newline at end of file
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java?rev=980297&r1=980296&r2=980297&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java Thu Jul 29 02:41:14 2010
@@ -65,10 +65,8 @@ public final class TaskFactory {
.add(new taskTuple<ExplainWork>(ExplainWork.class, ExplainTask.class));
taskvec.add(new taskTuple<ConditionalWork>(ConditionalWork.class,
ConditionalTask.class));
- // we are taking this out to allow us to instantiate either MapRedTask or
- // ExecDriver dynamically at run time based on configuration
- // taskvec.add(new taskTuple<mapredWork>(mapredWork.class,
- // ExecDriver.class));
+ taskvec.add(new taskTuple<MapredWork>(MapredWork.class,
+ MapRedTask.class));
}
private static ThreadLocal<Integer> tid = new ThreadLocal<Integer>() {
@@ -104,28 +102,6 @@ public final class TaskFactory {
}
}
- if (workClass == MapredWork.class) {
-
- boolean viachild = conf.getBoolVar(HiveConf.ConfVars.SUBMITVIACHILD);
-
- try {
-
- // in local mode - or if otherwise so configured - always submit
- // jobs via separate jvm
- Task<T> ret = null;
- if (conf.getVar(HiveConf.ConfVars.HADOOPJT).equals("local") || viachild) {
- ret = (Task<T>) MapRedTask.class.newInstance();
- } else {
- ret = (Task<T>) ExecDriver.class.newInstance();
- }
- ret.setId("Stage-" + Integer.toString(getAndIncrementId()));
- return ret;
- } catch (Exception e) {
- throw new RuntimeException(e.getMessage(), e);
- }
-
- }
-
throw new RuntimeException("No task for work class " + workClass.getName());
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=980297&r1=980296&r2=980297&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Thu Jul 29 02:41:14 2010
@@ -65,10 +65,13 @@ import org.apache.hadoop.fs.FSDataOutput
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Order;
import org.apache.hadoop.hive.ql.QueryPlan;
+import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat;
import org.apache.hadoop.hive.ql.io.RCFile;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -127,7 +130,7 @@ public final class Utilities {
public static void clearMapRedWork(Configuration job) {
try {
Path planPath = new Path(HiveConf.getVar(job, HiveConf.ConfVars.PLAN));
- FileSystem fs = FileSystem.get(job);
+ FileSystem fs = planPath.getFileSystem(job);
if (fs.exists(planPath)) {
try {
fs.delete(planPath, true);
@@ -1332,4 +1335,79 @@ public final class Utilities {
}
}
+ /**
+ * Calculate the total size of input files.
+ *
+ * @param job the hadoop job conf.
+ * @param work map reduce job plan
+ * @param filter filter to apply to the input paths before calculating size
+ * @return the summary of all the input paths.
+ * @throws IOException
+ */
+ public static ContentSummary getInputSummary
+ (Context ctx, MapredWork work, PathFilter filter) throws IOException {
+
+ long[] summary = {0, 0, 0};
+
+ // For each input path, calculate the total size.
+ for (String path : work.getPathToAliases().keySet()) {
+ try {
+ Path p = new Path(path);
+
+ if(filter != null && !filter.accept(p))
+ continue;
+
+ ContentSummary cs = ctx.getCS(path);
+ if (cs == null) {
+ FileSystem fs = p.getFileSystem(ctx.getConf());
+ cs = fs.getContentSummary(p);
+ ctx.addCS(path, cs);
+ }
+
+ summary[0] += cs.getLength();
+ summary[1] += cs.getFileCount();
+ summary[2] += cs.getDirectoryCount();
+
+ } catch (IOException e) {
+ LOG.info("Cannot get size of " + path + ". Safely ignored.");
+ if (path != null)
+ ctx.addCS(path, new ContentSummary(0, 0, 0));
+ }
+ }
+ return new ContentSummary(summary[0], summary[1], summary[2]);
+ }
+
+ public static boolean isEmptyPath(JobConf job, String path) throws Exception {
+ Path dirPath = new Path(path);
+ FileSystem inpFs = dirPath.getFileSystem(job);
+
+ if (inpFs.exists(dirPath)) {
+ FileStatus[] fStats = inpFs.listStatus(dirPath);
+ if (fStats.length > 0) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ public static List<ExecDriver> getMRTasks (List<Task<? extends Serializable>> tasks) {
+ List<ExecDriver> mrTasks = new ArrayList<ExecDriver> ();
+ if(tasks != null)
+ getMRTasks(tasks, mrTasks);
+ return mrTasks;
+ }
+
+ private static void getMRTasks (List<Task<? extends Serializable>> tasks,
+ List<ExecDriver> mrTasks) {
+ for (Task<? extends Serializable> task : tasks) {
+ if (task instanceof ExecDriver && !mrTasks.contains((ExecDriver)task))
+ mrTasks.add((ExecDriver)task);
+
+ if (task instanceof ConditionalTask)
+ getMRTasks(((ConditionalTask)task).getListTasks(), mrTasks);
+
+ if (task.getChildTasks() != null)
+ getMRTasks(task.getChildTasks(), mrTasks);
+ }
+ }
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketMapJoinOptimizer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketMapJoinOptimizer.java?rev=980297&r1=980296&r2=980297&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketMapJoinOptimizer.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketMapJoinOptimizer.java Thu Jul 29 02:41:14 2010
@@ -273,7 +273,7 @@ public class BucketMapJoinOptimizer impl
aliasToBucketNumberMapping.put(alias, num);
List<String> fileNames = new ArrayList<String>();
try {
- FileSystem fs = FileSystem.get(this.pGraphContext.getConf());
+ FileSystem fs = FileSystem.get(tbl.getDataLocation(), this.pGraphContext.getConf());
FileStatus[] files = fs.listStatus(new Path(tbl.getDataLocation().toString()));
if(files != null) {
for(FileStatus file : files) {
@@ -412,7 +412,7 @@ public class BucketMapJoinOptimizer impl
throws SemanticException {
List<String> fileNames = new ArrayList<String>();
try {
- FileSystem fs = FileSystem.get(this.pGraphContext.getConf());
+ FileSystem fs = FileSystem.get(part.getDataLocation(), this.pGraphContext.getConf());
FileStatus[] files = fs.listStatus(new Path(part.getDataLocation()
.toString()));
if (files != null) {
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java?rev=980297&r1=980296&r2=980297&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java Thu Jul 29 02:41:14 2010
@@ -587,7 +587,8 @@ public final class GenMapRedUtils {
continue;
}
String path = p.toString();
- LOG.debug("Adding " + path + " of table" + alias_id);
+ if (LOG.isDebugEnabled())
+ LOG.debug("Adding " + path + " of table" + alias_id);
partDir.add(p);
try {
@@ -615,7 +616,8 @@ public final class GenMapRedUtils {
}
plan.getPathToAliases().get(path).add(alias_id);
plan.getPathToPartitionInfo().put(path, prtDesc);
- LOG.debug("Information added for path " + path);
+ if (LOG.isDebugEnabled())
+ LOG.debug("Information added for path " + path);
}
assert plan.getAliasToWork().get(alias_id) == null;
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java?rev=980297&r1=980296&r2=980297&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java Thu Jul 29 02:41:14 2010
@@ -238,13 +238,15 @@ public class PartitionPruner implements
} else {
Partition part = Hive.get().getPartition(tab, partSpec,
Boolean.FALSE);
+ String state = "retained";
if (Boolean.TRUE.equals(r)) {
- LOG.debug("retained partition: " + partSpec);
true_parts.add(part);
} else {
- LOG.debug("unknown partition: " + partSpec);
unkn_parts.add(part);
+ state = "unknown";
}
+ if (LOG.isDebugEnabled())
+ LOG.debug(state + " partition: " + partSpec);
}
} else {
// is there is no parition pruning, all of them are needed
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=980297&r1=980296&r2=980297&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Thu Jul 29 02:41:14 2010
@@ -24,6 +24,7 @@ import static org.apache.hadoop.hive.ser
import static org.apache.hadoop.util.StringUtils.stringifyException;
import java.io.Serializable;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -39,22 +40,26 @@ import java.util.regex.PatternSyntaxExce
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Order;
+import org.apache.hadoop.hive.ql.Context;
+import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.exec.AbstractMapJoinOperator;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.ConditionalTask;
-import org.apache.hadoop.hive.ql.exec.ExecDriver;
import org.apache.hadoop.hive.ql.exec.FetchTask;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
import org.apache.hadoop.hive.ql.exec.FunctionInfo;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.GroupByOperator;
import org.apache.hadoop.hive.ql.exec.JoinOperator;
+import org.apache.hadoop.hive.ql.exec.ExecDriver;
import org.apache.hadoop.hive.ql.exec.MapRedTask;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.OperatorFactory;
@@ -1187,8 +1192,9 @@ public class SemanticAnalyzer extends Ba
new FilterDesc(genExprNodeDesc(condn, inputRR), false), new RowSchema(
inputRR.getColumnInfos()), input), inputRR);
- LOG.debug("Created Filter Plan for " + qb.getId() + " row schema: "
- + inputRR.toString());
+ if (LOG.isDebugEnabled())
+ LOG.debug("Created Filter Plan for " + qb.getId() + " row schema: "
+ + inputRR.toString());
return output;
}
@@ -1685,14 +1691,19 @@ public class SemanticAnalyzer extends Ba
ASTNode selExprList = qb.getParseInfo().getSelForClause(dest);
Operator<?> op = genSelectPlan(selExprList, qb, input);
- LOG.debug("Created Select Plan for clause: " + dest);
+
+ if (LOG.isDebugEnabled())
+ LOG.debug("Created Select Plan for clause: " + dest);
+
return op;
}
@SuppressWarnings("nls")
private Operator<?> genSelectPlan(ASTNode selExprList, QB qb,
Operator<?> input) throws SemanticException {
- LOG.debug("tree: " + selExprList.toStringTree());
+
+ if (LOG.isDebugEnabled())
+ LOG.debug("tree: " + selExprList.toStringTree());
ArrayList<ExprNodeDesc> col_list = new ArrayList<ExprNodeDesc>();
RowResolver out_rwsch = new RowResolver();
@@ -1770,8 +1781,10 @@ public class SemanticAnalyzer extends Ba
assert (false);
}
}
- LOG.debug("UDTF table alias is " + udtfTableAlias);
- LOG.debug("UDTF col aliases are " + udtfColAliases);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("UDTF table alias is " + udtfTableAlias);
+ LOG.debug("UDTF col aliases are " + udtfColAliases);
+ }
}
// The list of expressions after SELECT or SELECT TRANSFORM.
@@ -1784,7 +1797,8 @@ public class SemanticAnalyzer extends Ba
exprList = selExprList;
}
- LOG.debug("genSelectPlan: input = " + inputRR.toString());
+ if (LOG.isDebugEnabled())
+ LOG.debug("genSelectPlan: input = " + inputRR.toString());
// For UDTF's, skip the function name to get the expressions
int startPosn = isUDTF ? posn + 1 : posn;
@@ -1894,7 +1908,8 @@ public class SemanticAnalyzer extends Ba
output = genUDTFPlan(genericUDTF, udtfTableAlias, udtfColAliases, qb,
output);
}
- LOG.debug("Created Select Plan row schema: " + out_rwsch.toString());
+ if (LOG.isDebugEnabled())
+ LOG.debug("Created Select Plan row schema: " + out_rwsch.toString());
return output;
}
@@ -3494,8 +3509,9 @@ public class SemanticAnalyzer extends Ba
.mapDirToFop(ltd.getSourceDir(), (FileSinkOperator)output);
}
- LOG.debug("Created FileSink Plan for clause: " + dest + "dest_path: "
- + dest_path + " row schema: " + inputRR.toString());
+ if (LOG.isDebugEnabled())
+ LOG.debug("Created FileSink Plan for clause: " + dest + "dest_path: "
+ + dest_path + " row schema: " + inputRR.toString());
return output;
}
@@ -3648,8 +3664,9 @@ public class SemanticAnalyzer extends Ba
new LimitDesc(limit), new RowSchema(inputRR.getColumnInfos()), input),
inputRR);
- LOG.debug("Created LimitOperator Plan for clause: " + dest
- + " row schema: " + inputRR.toString());
+ if (LOG.isDebugEnabled())
+ LOG.debug("Created LimitOperator Plan for clause: " + dest
+ + " row schema: " + inputRR.toString());
return limitMap;
}
@@ -3676,8 +3693,9 @@ public class SemanticAnalyzer extends Ba
throw new SemanticException(ErrorMsg.UDTF_LATERAL_VIEW.getMsg());
}
- LOG.debug("Table alias: " + outputTableAlias + " Col aliases: "
- + colAliases);
+ if (LOG.isDebugEnabled())
+ LOG.debug("Table alias: " + outputTableAlias + " Col aliases: "
+ + colAliases);
// Use the RowResolver from the input operator to generate a input
// ObjectInspector that can be used to initialize the UDTF. Then, the
@@ -3908,8 +3926,9 @@ public class SemanticAnalyzer extends Ba
Utilities.ReduceField.VALUE.toString(), "", false)), new RowSchema(
out_rwsch.getColumnInfos()), interim), out_rwsch);
- LOG.debug("Created ReduceSink Plan for table: " + tab.getTableName() + " row schema: "
- + out_rwsch.toString());
+ if (LOG.isDebugEnabled())
+ LOG.debug("Created ReduceSink Plan for table: " + tab.getTableName() +
+ " row schema: " + out_rwsch.toString());
return output;
}
@@ -4018,8 +4037,9 @@ public class SemanticAnalyzer extends Ba
Utilities.ReduceField.VALUE.toString(), "", false)), new RowSchema(
out_rwsch.getColumnInfos()), interim), out_rwsch);
- LOG.debug("Created ReduceSink Plan for clause: " + dest + " row schema: "
- + out_rwsch.toString());
+ if (LOG.isDebugEnabled())
+ LOG.debug("Created ReduceSink Plan for clause: " + dest + " row schema: "
+ + out_rwsch.toString());
return output;
}
@@ -5132,7 +5152,9 @@ public class SemanticAnalyzer extends Ba
}
}
- LOG.debug("Created Body Plan for Query Block " + qb.getId());
+ if (LOG.isDebugEnabled())
+ LOG.debug("Created Body Plan for Query Block " + qb.getId());
+
return curr;
}
@@ -5517,7 +5539,9 @@ public class SemanticAnalyzer extends Ba
}
Operator output = putOpInsertMap(tableOp, rwsch);
- LOG.debug("Created Table Plan for " + alias + " " + tableOp.toString());
+
+ if (LOG.isDebugEnabled())
+ LOG.debug("Created Table Plan for " + alias + " " + tableOp.toString());
return output;
}
@@ -5584,7 +5608,9 @@ public class SemanticAnalyzer extends Ba
}
Operator bodyOpInfo = genBodyPlan(qb, srcOpInfo);
- LOG.debug("Created Plan for Query Block " + qb.getId());
+
+ if (LOG.isDebugEnabled())
+ LOG.debug("Created Plan for Query Block " + qb.getId());
this.qb = qb;
return bodyOpInfo;
@@ -5917,6 +5943,8 @@ public class SemanticAnalyzer extends Ba
}
}
+ decideExecMode(rootTasks, ctx);
+
if (qb.isCTAS()) {
// generate a DDL task and make it a dependent task of the leaf
CreateTableDesc crtTblDesc = qb.getTableDesc();
@@ -5966,7 +5994,7 @@ public class SemanticAnalyzer extends Ba
// loop over all the tasks recursviely
private void generateCountersTask(Task<? extends Serializable> task) {
- if ((task instanceof MapRedTask) || (task instanceof ExecDriver)) {
+ if (task instanceof ExecDriver) {
HashMap<String, Operator<? extends Serializable>> opMap = ((MapredWork) task
.getWork()).getAliasToWork();
if (!opMap.isEmpty()) {
@@ -6016,7 +6044,7 @@ public class SemanticAnalyzer extends Ba
// loop over all the tasks recursviely
private void breakTaskTree(Task<? extends Serializable> task) {
- if ((task instanceof MapRedTask) || (task instanceof ExecDriver)) {
+ if (task instanceof ExecDriver) {
HashMap<String, Operator<? extends Serializable>> opMap = ((MapredWork) task
.getWork()).getAliasToWork();
if (!opMap.isEmpty()) {
@@ -6059,7 +6087,7 @@ public class SemanticAnalyzer extends Ba
// loop over all the tasks recursviely
private void setKeyDescTaskTree(Task<? extends Serializable> task) {
- if ((task instanceof MapRedTask) || (task instanceof ExecDriver)) {
+ if (task instanceof ExecDriver) {
MapredWork work = (MapredWork) task.getWork();
work.deriveExplainAttributes();
HashMap<String, Operator<? extends Serializable>> opMap = work
@@ -6397,8 +6425,6 @@ public class SemanticAnalyzer extends Ba
@Override
public void validate() throws SemanticException {
- // Check if the plan contains atleast one path.
-
// validate all tasks
for (Task<? extends Serializable> rootTask : rootTasks) {
validate(rootTask);
@@ -6407,13 +6433,6 @@ public class SemanticAnalyzer extends Ba
private void validate(Task<? extends Serializable> task)
throws SemanticException {
- if ((task instanceof MapRedTask) || (task instanceof ExecDriver)) {
- task.getWork();
-
- // If the plan does not contain any path, an empty file
- // will be added by ExecDriver at execute time
- }
-
if (task.getChildTasks() == null) {
return;
}
@@ -6857,4 +6876,81 @@ public class SemanticAnalyzer extends Ba
}
}
}
+
+ private void decideExecMode(List<Task<? extends Serializable>> rootTasks, Context ctx)
+ throws SemanticException {
+
+ // bypass for explain queries for now
+ if (ctx.getExplain())
+ return;
+
+ // user has told us to run in local mode or doesn't want auto-local mode
+ if (ctx.isLocalOnlyExecutionMode() ||
+ !conf.getBoolVar(HiveConf.ConfVars.LOCALMODEAUTO))
+ return;
+
+ final Context lCtx = ctx;
+ PathFilter p = new PathFilter () {
+ public boolean accept(Path file) {
+ return !lCtx.isMRTmpFileURI(file.toUri().getPath());
+ }
+ };
+ List<ExecDriver> mrtasks = Utilities.getMRTasks(rootTasks);
+
+ // map-reduce jobs will be run locally based on data size
+ // first find out if any of the jobs needs to run non-locally
+ boolean hasNonLocalJob = false;
+ for (ExecDriver mrtask: mrtasks) {
+ try {
+ ContentSummary inputSummary = Utilities.getInputSummary
+ (ctx, (MapredWork)mrtask.getWork(), p);
+ int numReducers = getNumberOfReducers(mrtask.getWork(), conf);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Task: " + mrtask.getId() + ", Summary: " +
+ inputSummary.getLength() + "," + inputSummary.getFileCount() + ","
+ + numReducers);
+ }
+
+ if(MapRedTask.isEligibleForLocalMode(conf, inputSummary, numReducers) != null) {
+ hasNonLocalJob = true;
+ break;
+ }
+ } catch (IOException e) {
+ throw new SemanticException (e);
+ }
+ }
+
+ if(!hasNonLocalJob) {
+ // none of the mapred tasks needs to be run locally. That means that the
+ // query can be executed entirely in local mode. Save the current tracker
+ // value and restore it when done
+ ctx.setOriginalTracker(conf.getVar(HiveConf.ConfVars.HADOOPJT));
+ conf.setVar(HiveConf.ConfVars.HADOOPJT, "local");
+ console.printInfo("Automatically selecting local only mode for query");
+
+ // If all the tasks can be run locally, we can use local disk for
+ // storing intermediate data.
+
+ /**
+ * This code is commented out pending further testing/development
+ * for (Task<? extends Serializable> t: rootTasks)
+ * t.localizeMRTmpFiles(ctx);
+ */
+ }
+ }
+
+ /**
+ * Make a best guess at trying to find the number of reducers
+ */
+ private static int getNumberOfReducers(MapredWork mrwork, HiveConf conf) {
+ if (mrwork.getReducer() == null)
+ return 0;
+
+ if (mrwork.getNumReduceTasks() >= 0)
+ return mrwork.getNumReduceTasks();
+
+ return conf.getIntVar(HiveConf.ConfVars.HADOOPNUMREDUCERS);
+ }
+
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapredLocalWork.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapredLocalWork.java?rev=980297&r1=980296&r2=980297&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapredLocalWork.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapredLocalWork.java Thu Jul 29 02:41:14 2010
@@ -28,6 +28,7 @@ import java.util.Map.Entry;
import org.apache.hadoop.hive.ql.exec.BucketMatcher;
import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.fs.Path;
/**
* MapredLocalWork.
@@ -166,9 +167,7 @@ public class MapredLocalWork implements
private String getBaseFileName (String path) {
try {
- URI uri = new URI(path);
- File file = new File(uri);
- return file.getName();
+ return ((new Path(path)).getName());
} catch (Exception ex) {
// This could be due to either URI syntax error or File constructor
// illegal arg; we don't really care which one it is.
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapredWork.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapredWork.java?rev=980297&r1=980296&r2=980297&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapredWork.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapredWork.java Thu Jul 29 02:41:14 2010
@@ -321,5 +321,4 @@ public class MapredWork implements Seria
public void setInputformat(String inputformat) {
this.inputformat = inputformat;
}
-
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java?rev=980297&r1=980296&r2=980297&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java Thu Jul 29 02:41:14 2010
@@ -30,6 +30,7 @@ import org.apache.hadoop.hive.ql.io.Hive
import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.fs.Path;
/**
* PartitionDesc.
@@ -249,12 +250,11 @@ public class PartitionDesc implements Se
return;
}
try {
- URI uri = new URI(path);
- File file = new File(uri);
- baseFileName = file.getName();
+ Path p = new Path(path);
+ baseFileName = p.getName();
} catch (Exception ex) {
- // This could be due to either URI syntax error or File constructor
- // illegal arg; we don't really care which one it is.
+ // don't really care about the exception. the goal is to capture the
+ // the last component at the minimum - so set to the complete path
baseFileName = path;
}
}
Modified: hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java?rev=980297&r1=980296&r2=980297&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (original)
+++ hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java Thu Jul 29 02:41:14 2010
@@ -334,11 +334,11 @@ public class QTestUtil {
private void runLoadCmd(String loadCmd) throws Exception {
int ecode = 0;
ecode = drv.run(loadCmd).getResponseCode();
+ drv.close();
if (ecode != 0) {
throw new Exception("load command: " + loadCmd
+ " failed with exit code= " + ecode);
}
-
return;
}
@@ -366,7 +366,6 @@ public class QTestUtil {
IgnoreKeyTextOutputFormat.class);
Path fpath;
- Path newfpath;
HashMap<String, String> part_spec = new HashMap<String, String>();
for (String ds : new String[] {"2008-04-08", "2008-04-09"}) {
for (String hr : new String[] {"11", "12"}) {
@@ -376,11 +375,8 @@ public class QTestUtil {
// System.out.println("Loading partition with spec: " + part_spec);
// db.createPartition(srcpart, part_spec);
fpath = new Path(testFiles, "kv1.txt");
- newfpath = new Path(tmppath, "kv1.txt");
- fs.copyFromLocalFile(false, true, fpath, newfpath);
- fpath = newfpath;
// db.loadPartition(fpath, srcpart.getName(), part_spec, true);
- runLoadCmd("LOAD DATA INPATH '" + newfpath.toString()
+ runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toString()
+ "' OVERWRITE INTO TABLE srcpart PARTITION (ds='" + ds + "',hr='"
+ hr + "')");
}
@@ -392,9 +388,7 @@ public class QTestUtil {
// IgnoreKeyTextOutputFormat.class, 2, bucketCols);
for (String fname : new String[] {"srcbucket0.txt", "srcbucket1.txt"}) {
fpath = new Path(testFiles, fname);
- newfpath = new Path(tmppath, fname);
- fs.copyFromLocalFile(false, true, fpath, newfpath);
- runLoadCmd("LOAD DATA INPATH '" + newfpath.toString()
+ runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toString()
+ "' INTO TABLE srcbucket");
}
@@ -405,9 +399,7 @@ public class QTestUtil {
for (String fname : new String[] {"srcbucket20.txt", "srcbucket21.txt",
"srcbucket22.txt", "srcbucket23.txt"}) {
fpath = new Path(testFiles, fname);
- newfpath = new Path(tmppath, fname);
- fs.copyFromLocalFile(false, true, fpath, newfpath);
- runLoadCmd("LOAD DATA INPATH '" + newfpath.toString()
+ runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toString()
+ "' INTO TABLE srcbucket2");
}
@@ -435,40 +427,25 @@ public class QTestUtil {
// load the input data into the src table
fpath = new Path(testFiles, "kv1.txt");
- newfpath = new Path(tmppath, "kv1.txt");
- fs.copyFromLocalFile(false, true, fpath, newfpath);
- // db.loadTable(newfpath, "src", false);
- runLoadCmd("LOAD DATA INPATH '" + newfpath.toString() + "' INTO TABLE src");
+ runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toString() + "' INTO TABLE src");
// load the input data into the src table
fpath = new Path(testFiles, "kv3.txt");
- newfpath = new Path(tmppath, "kv3.txt");
- fs.copyFromLocalFile(false, true, fpath, newfpath);
- // db.loadTable(newfpath, "src1", false);
- runLoadCmd("LOAD DATA INPATH '" + newfpath.toString() + "' INTO TABLE src1");
+ runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toString() + "' INTO TABLE src1");
// load the input data into the src_sequencefile table
fpath = new Path(testFiles, "kv1.seq");
- newfpath = new Path(tmppath, "kv1.seq");
- fs.copyFromLocalFile(false, true, fpath, newfpath);
- // db.loadTable(newfpath, "src_sequencefile", true);
- runLoadCmd("LOAD DATA INPATH '" + newfpath.toString()
+ runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toString()
+ "' INTO TABLE src_sequencefile");
// load the input data into the src_thrift table
fpath = new Path(testFiles, "complex.seq");
- newfpath = new Path(tmppath, "complex.seq");
- fs.copyFromLocalFile(false, true, fpath, newfpath);
- // db.loadTable(newfpath, "src_thrift", true);
- runLoadCmd("LOAD DATA INPATH '" + newfpath.toString()
+ runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toString()
+ "' INTO TABLE src_thrift");
// load the json data into the src_json table
fpath = new Path(testFiles, "json.txt");
- newfpath = new Path(tmppath, "json.txt");
- fs.copyFromLocalFile(false, true, fpath, newfpath);
- // db.loadTable(newfpath, "src_json", false);
- runLoadCmd("LOAD DATA INPATH '" + newfpath.toString()
+ runLoadCmd("LOAD DATA LOCAL INPATH '" + fpath.toString()
+ "' INTO TABLE src_json");
}
@@ -709,6 +686,7 @@ public class QTestUtil {
cmdArray[3] = "\\(\\(<java version=\".*\" class=\"java.beans.XMLDecoder\">\\)"
+ "\\|\\(<string>.*/tmp/.*</string>\\)"
+ "\\|\\(<string>file:.*</string>\\)"
+ + "\\|\\(<string>pfile:.*</string>\\)"
+ "\\|\\(<string>[0-9]\\{10\\}</string>\\)"
+ "\\|\\(<string>/.*/warehouse/.*</string>\\)\\)";
cmdArray[4] = outf.getPath();
@@ -864,6 +842,7 @@ public class QTestUtil {
cmdArray = new String[] {
"diff", "-a",
"-I", "file:",
+ "-I", "pfile:",
"-I", "/tmp/",
"-I", "invalidscheme:",
"-I", "lastUpdateTime",
Modified: hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java?rev=980297&r1=980296&r2=980297&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java (original)
+++ hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java Thu Jul 29 02:41:14 2010
@@ -48,13 +48,11 @@ import org.apache.thrift.protocol.TBinar
public class TestHive extends TestCase {
private Hive hm;
private HiveConf hiveConf;
- private FileSystem fs;
@Override
protected void setUp() throws Exception {
super.setUp();
hiveConf = new HiveConf(this.getClass());
- fs = FileSystem.get(hiveConf);
try {
hm = Hive.get(hiveConf);
} catch (Exception e) {
@@ -308,6 +306,7 @@ public class TestHive extends TestCase {
assertNotNull(table1);
assertEquals(table1Name, table1.getTableName());
+ FileSystem fs = table1.getPath().getFileSystem(hiveConf);
assertTrue(fs.exists(table1.getPath()));
// and test dropping this specific table
hm.dropTable(dbName, table1Name);
Added: hadoop/hive/trunk/ql/src/test/queries/clientnegative/autolocal1.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientnegative/autolocal1.q?rev=980297&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientnegative/autolocal1.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientnegative/autolocal1.q Thu Jul 29 02:41:14 2010
@@ -0,0 +1,5 @@
+set mapred.job.tracker=abracadabra;
+set hive.exec.mode.local.auto.inputbytes.max=1;
+set hive.exec.mode.local.auto=true;
+
+SELECT key FROM src;
Modified: hadoop/hive/trunk/ql/src/test/queries/clientpositive/ctas.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/ctas.q?rev=980297&r1=980296&r2=980297&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/ctas.q (original)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/ctas.q Thu Jul 29 02:41:14 2010
@@ -45,6 +45,9 @@ select * from nzhang_ctas4;
explain extended create table nzhang_ctas5 row format delimited fields terminated by ',' lines terminated by '\012' stored as textfile as select key, value from src sort by key, value limit 10;
+set mapred.job.tracker=does.notexist.com:666;
+set hive.exec.mode.local.auto=true;
+
create table nzhang_ctas5 row format delimited fields terminated by ',' lines terminated by '\012' stored as textfile as select key, value from src sort by key, value limit 10;
create table nzhang_ctas6 (key string, `to` string);
Modified: hadoop/hive/trunk/ql/src/test/queries/clientpositive/input12.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/input12.q?rev=980297&r1=980296&r2=980297&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/input12.q (original)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/input12.q Thu Jul 29 02:41:14 2010
@@ -1,3 +1,6 @@
+set mapred.job.tracker=does.notexist.com:666;
+set hive.exec.mode.local.auto=true;
+
CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE;
CREATE TABLE dest2(key INT, value STRING) STORED AS TEXTFILE;
CREATE TABLE dest3(key INT) PARTITIONED BY(ds STRING, hr STRING) STORED AS TEXTFILE;
Modified: hadoop/hive/trunk/ql/src/test/queries/clientpositive/input39.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/input39.q?rev=980297&r1=980296&r2=980297&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/input39.q (original)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/input39.q Thu Jul 29 02:41:14 2010
@@ -15,6 +15,8 @@ select key, value from src;
set hive.test.mode=true;
set hive.mapred.mode=strict;
+set mapred.job.tracker=does.notexist.com:666;
+set hive.exec.mode.local.auto=true;
explain
select count(1) from t1 join t2 on t1.key=t2.key where t1.ds='1' and t2.ds='1';
@@ -22,6 +24,7 @@ select count(1) from t1 join t2 on t1.ke
select count(1) from t1 join t2 on t1.key=t2.key where t1.ds='1' and t2.ds='1';
set hive.test.mode=false;
+set mapred.job.tracker;
Modified: hadoop/hive/trunk/ql/src/test/queries/clientpositive/insertexternal1.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/insertexternal1.q?rev=980297&r1=980296&r2=980297&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/insertexternal1.q (original)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/insertexternal1.q Thu Jul 29 02:41:14 2010
@@ -5,7 +5,7 @@ create table texternal(key string, val s
!rm -fr /tmp/texternal;
!mkdir -p /tmp/texternal/2008-01-01;
-alter table texternal add partition (insertdate='2008-01-01') location 'file:///tmp/texternal/2008-01-01';
+alter table texternal add partition (insertdate='2008-01-01') location 'pfile:///tmp/texternal/2008-01-01';
from src insert overwrite table texternal partition (insertdate='2008-01-01') select *;
select * from texternal where insertdate='2008-01-01';
Modified: hadoop/hive/trunk/ql/src/test/queries/clientpositive/join14.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/join14.q?rev=980297&r1=980296&r2=980297&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/join14.q (original)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/join14.q Thu Jul 29 02:41:14 2010
@@ -1,5 +1,8 @@
CREATE TABLE dest1(c1 INT, c2 STRING) STORED AS TEXTFILE;
+set mapred.job.tracker=does.notexist.com:666;
+set hive.exec.mode.local.auto=true;
+
EXPLAIN
FROM src JOIN srcpart ON src.key = srcpart.key AND srcpart.ds = '2008-04-08' and src.key > 100
INSERT OVERWRITE TABLE dest1 SELECT src.key, srcpart.value;
Added: hadoop/hive/trunk/ql/src/test/results/clientnegative/autolocal1.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientnegative/autolocal1.q.out?rev=980297&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientnegative/autolocal1.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientnegative/autolocal1.q.out Thu Jul 29 02:41:14 2010
@@ -0,0 +1,6 @@
+PREHOOK: query: SELECT key FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: file:/data/users/jssarma/hive_trunk/build/ql/scratchdir/hive_2010-07-25_01-23-08_631_3140637565055726778/-mr-10000
+Job Submission failed with exception 'java.lang.RuntimeException(Not a host:port pair: abracadabra)'
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.MapRedTask
Modified: hadoop/hive/trunk/ql/src/test/results/clientnegative/fs_default_name1.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientnegative/fs_default_name1.q.out?rev=980297&r1=980296&r2=980297&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientnegative/fs_default_name1.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientnegative/fs_default_name1.q.out Thu Jul 29 02:41:14 2010
@@ -1,15 +1,19 @@
-FAILED: Hive Internal Error: java.lang.RuntimeException(Error while making local scratch directory - check filesystem config (java.net.URISyntaxException: Illegal character in scheme name at index 0: 'http://www.example.com))
-java.lang.RuntimeException: Error while making local scratch directory - check filesystem config (java.net.URISyntaxException: Illegal character in scheme name at index 0: 'http://www.example.com)
- at org.apache.hadoop.hive.ql.Context.getLocalScratchDir(Context.java:225)
- at org.apache.hadoop.hive.ql.Context.getLocalTmpFileURI(Context.java:293)
- at org.apache.hadoop.hive.ql.parse.DDLSemanticAnalyzer.analyzeInternal(DDLSemanticAnalyzer.java:102)
+FAILED: Hive Internal Error: java.lang.IllegalArgumentException(null)
+java.lang.IllegalArgumentException
+ at java.net.URI.create(URI.java:842)
+ at org.apache.hadoop.fs.FileSystem.getDefaultUri(FileSystem.java:103)
+ at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:184)
+ at org.apache.hadoop.fs.FileSystem.getLocal(FileSystem.java:167)
+ at org.apache.hadoop.hive.ql.Context.getLocalScratchDir(Context.java:157)
+ at org.apache.hadoop.hive.ql.Context.getLocalTmpFileURI(Context.java:273)
+ at org.apache.hadoop.hive.ql.parse.DDLSemanticAnalyzer.analyzeInternal(DDLSemanticAnalyzer.java:114)
at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:126)
- at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:304)
- at org.apache.hadoop.hive.ql.Driver.run(Driver.java:377)
+ at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:301)
+ at org.apache.hadoop.hive.ql.Driver.run(Driver.java:376)
at org.apache.hadoop.hive.cli.CliDriver.processCmd(CliDriver.java:138)
at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:197)
- at org.apache.hadoop.hive.ql.QTestUtil.executeClient(QTestUtil.java:504)
- at org.apache.hadoop.hive.cli.TestNegativeCliDriver.testNegativeCliDriver_fs_default_name1(TestNegativeCliDriver.java:54)
+ at org.apache.hadoop.hive.ql.QTestUtil.executeClient(QTestUtil.java:552)
+ at org.apache.hadoop.hive.cli.TestNegativeCliDriver.testNegativeCliDriver_fs_default_name1(TestNegativeCliDriver.java:2187)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
@@ -22,17 +26,9 @@ java.lang.RuntimeException: Error while
at junit.framework.TestCase.run(TestCase.java:118)
at junit.framework.TestSuite.runTest(TestSuite.java:208)
at junit.framework.TestSuite.run(TestSuite.java:203)
- at org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.run(JUnitTestRunner.java:420)
- at org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.launch(JUnitTestRunner.java:911)
- at org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.main(JUnitTestRunner.java:768)
-Caused by: java.lang.IllegalArgumentException
- at java.net.URI.create(URI.java:842)
- at org.apache.hadoop.fs.FileSystem.getDefaultUri(FileSystem.java:103)
- at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:184)
- at org.apache.hadoop.fs.FileSystem.getLocal(FileSystem.java:167)
- at org.apache.hadoop.hive.ql.Context.makeLocalScratchDir(Context.java:165)
- at org.apache.hadoop.hive.ql.Context.getLocalScratchDir(Context.java:219)
- ... 24 more
+ at org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.run(JUnitTestRunner.java:422)
+ at org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.launch(JUnitTestRunner.java:931)
+ at org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.main(JUnitTestRunner.java:785)
Caused by: java.net.URISyntaxException: Illegal character in scheme name at index 0: 'http://www.example.com
at java.net.URI$Parser.fail(URI.java:2809)
at java.net.URI$Parser.checkChars(URI.java:2982)
@@ -40,5 +36,5 @@ Caused by: java.net.URISyntaxException:
at java.net.URI$Parser.parse(URI.java:3008)
at java.net.URI.<init>(URI.java:578)
at java.net.URI.create(URI.java:840)
- ... 29 more
+ ... 28 more
Modified: hadoop/hive/trunk/ql/src/test/results/clientnegative/fs_default_name2.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientnegative/fs_default_name2.q.out?rev=980297&r1=980296&r2=980297&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientnegative/fs_default_name2.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientnegative/fs_default_name2.q.out Thu Jul 29 02:41:14 2010
@@ -1,16 +1,21 @@
-FAILED: Hive Internal Error: java.lang.RuntimeException(Error while making MR scratch directory - check filesystem config (java.net.URISyntaxException: Illegal character in scheme name at index 0: 'http://www.example.com))
-java.lang.RuntimeException: Error while making MR scratch directory - check filesystem config (java.net.URISyntaxException: Illegal character in scheme name at index 0: 'http://www.example.com)
- at org.apache.hadoop.hive.ql.Context.getMRScratchDir(Context.java:208)
- at org.apache.hadoop.hive.ql.Context.getMRTmpFileURI(Context.java:284)
- at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.getMetaData(SemanticAnalyzer.java:806)
- at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeInternal(SemanticAnalyzer.java:6041)
+FAILED: Hive Internal Error: java.lang.IllegalArgumentException(null)
+java.lang.IllegalArgumentException
+ at java.net.URI.create(URI.java:842)
+ at org.apache.hadoop.fs.FileSystem.getDefaultUri(FileSystem.java:103)
+ at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:184)
+ at org.apache.hadoop.fs.FileSystem.getLocal(FileSystem.java:167)
+ at org.apache.hadoop.hive.ql.Context.getLocalScratchDir(Context.java:157)
+ at org.apache.hadoop.hive.ql.Context.getMRScratchDir(Context.java:176)
+ at org.apache.hadoop.hive.ql.Context.getMRTmpFileURI(Context.java:238)
+ at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.getMetaData(SemanticAnalyzer.java:831)
+ at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeInternal(SemanticAnalyzer.java:6160)
at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:126)
- at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:304)
- at org.apache.hadoop.hive.ql.Driver.run(Driver.java:377)
+ at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:301)
+ at org.apache.hadoop.hive.ql.Driver.run(Driver.java:376)
at org.apache.hadoop.hive.cli.CliDriver.processCmd(CliDriver.java:138)
at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:197)
- at org.apache.hadoop.hive.ql.QTestUtil.executeClient(QTestUtil.java:504)
- at org.apache.hadoop.hive.cli.TestNegativeCliDriver.testNegativeCliDriver_fs_default_name2(TestNegativeCliDriver.java:54)
+ at org.apache.hadoop.hive.ql.QTestUtil.executeClient(QTestUtil.java:552)
+ at org.apache.hadoop.hive.cli.TestNegativeCliDriver.testNegativeCliDriver_fs_default_name2(TestNegativeCliDriver.java:2222)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
@@ -23,16 +28,9 @@ java.lang.RuntimeException: Error while
at junit.framework.TestCase.run(TestCase.java:118)
at junit.framework.TestSuite.runTest(TestSuite.java:208)
at junit.framework.TestSuite.run(TestSuite.java:203)
- at org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.run(JUnitTestRunner.java:420)
- at org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.launch(JUnitTestRunner.java:911)
- at org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.main(JUnitTestRunner.java:768)
-Caused by: java.lang.IllegalArgumentException
- at java.net.URI.create(URI.java:842)
- at org.apache.hadoop.fs.FileSystem.getDefaultUri(FileSystem.java:103)
- at org.apache.hadoop.hive.common.FileUtils.makeQualified(FileUtils.java:58)
- at org.apache.hadoop.hive.ql.Context.makeMRScratchDir(Context.java:128)
- at org.apache.hadoop.hive.ql.Context.getMRScratchDir(Context.java:202)
- ... 25 more
+ at org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.run(JUnitTestRunner.java:422)
+ at org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.launch(JUnitTestRunner.java:931)
+ at org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.main(JUnitTestRunner.java:785)
Caused by: java.net.URISyntaxException: Illegal character in scheme name at index 0: 'http://www.example.com
at java.net.URI$Parser.fail(URI.java:2809)
at java.net.URI$Parser.checkChars(URI.java:2982)
@@ -40,5 +38,5 @@ Caused by: java.net.URISyntaxException:
at java.net.URI$Parser.parse(URI.java:3008)
at java.net.URI.<init>(URI.java:578)
at java.net.URI.create(URI.java:840)
- ... 29 more
+ ... 30 more
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/binary_output_format.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/binary_output_format.q.out?rev=980297&r1=980296&r2=980297&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/binary_output_format.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/binary_output_format.q.out Thu Jul 29 02:41:14 2010
@@ -88,7 +88,7 @@ STAGE PLANS:
File Output Operator
compressed: false
GlobalTableId: 1
- directory: file:/mnt/vol/devrs004.snc1/jssarma/projects/hive_trunk/build/ql/scratchdir/hive_2010-07-21_11-08-59_521_6439138630072861096/10002
+ directory: pfile:/data/users/jssarma/hive_trunk/build/ql/scratchdir/hive_2010-07-25_11-37-07_143_545511266047909547/-ext-10002
NumFilesPerFileSink: 1
table:
input format: org.apache.hadoop.mapred.TextInputFormat
@@ -99,22 +99,22 @@ STAGE PLANS:
columns.types string
file.inputformat org.apache.hadoop.mapred.TextInputFormat
file.outputformat org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat
- location file:/mnt/vol/devrs004.snc1/jssarma/projects/hive_trunk/build/ql/test/data/warehouse/dest1
+ location pfile:/data/users/jssarma/hive_trunk/build/ql/test/data/warehouse/dest1
name dest1
serialization.ddl struct dest1 { string mydata}
serialization.format 1
serialization.last.column.takes.rest true
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- transient_lastDdlTime 1279735739
+ transient_lastDdlTime 1280083027
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: dest1
TotalFiles: 1
MultiFileSpray: false
Needs Tagging: false
Path -> Alias:
- file:/mnt/vol/devrs004.snc1/jssarma/projects/hive_trunk/build/ql/test/data/warehouse/src [src]
+ pfile:/data/users/jssarma/hive_trunk/build/ql/test/data/warehouse/src [src]
Path -> Partition:
- file:/mnt/vol/devrs004.snc1/jssarma/projects/hive_trunk/build/ql/test/data/warehouse/src
+ pfile:/data/users/jssarma/hive_trunk/build/ql/test/data/warehouse/src
Partition
base file name: src
input format: org.apache.hadoop.mapred.TextInputFormat
@@ -125,12 +125,12 @@ STAGE PLANS:
columns.types string:string
file.inputformat org.apache.hadoop.mapred.TextInputFormat
file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- location file:/mnt/vol/devrs004.snc1/jssarma/projects/hive_trunk/build/ql/test/data/warehouse/src
+ location pfile:/data/users/jssarma/hive_trunk/build/ql/test/data/warehouse/src
name src
serialization.ddl struct src { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- transient_lastDdlTime 1279735685
+ transient_lastDdlTime 1280082972
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
input format: org.apache.hadoop.mapred.TextInputFormat
@@ -141,12 +141,12 @@ STAGE PLANS:
columns.types string:string
file.inputformat org.apache.hadoop.mapred.TextInputFormat
file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- location file:/mnt/vol/devrs004.snc1/jssarma/projects/hive_trunk/build/ql/test/data/warehouse/src
+ location pfile:/data/users/jssarma/hive_trunk/build/ql/test/data/warehouse/src
name src
serialization.ddl struct src { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- transient_lastDdlTime 1279735685
+ transient_lastDdlTime 1280082972
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: src
name: src
@@ -158,14 +158,14 @@ STAGE PLANS:
Move Operator
files:
hdfs directory: true
- source: file:/mnt/vol/devrs004.snc1/jssarma/projects/hive_trunk/build/ql/scratchdir/hive_2010-07-21_11-08-59_521_6439138630072861096/10002
- destination: file:/mnt/vol/devrs004.snc1/jssarma/projects/hive_trunk/build/ql/scratchdir/hive_2010-07-21_11-08-59_521_6439138630072861096/10000
+ source: pfile:/data/users/jssarma/hive_trunk/build/ql/scratchdir/hive_2010-07-25_11-37-07_143_545511266047909547/-ext-10002
+ destination: pfile:/data/users/jssarma/hive_trunk/build/ql/scratchdir/hive_2010-07-25_11-37-07_143_545511266047909547/-ext-10000
Stage: Stage-0
Move Operator
tables:
replace: true
- source: file:/mnt/vol/devrs004.snc1/jssarma/projects/hive_trunk/build/ql/scratchdir/hive_2010-07-21_11-08-59_521_6439138630072861096/10000
+ source: pfile:/data/users/jssarma/hive_trunk/build/ql/scratchdir/hive_2010-07-25_11-37-07_143_545511266047909547/-ext-10000
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat
@@ -175,21 +175,21 @@ STAGE PLANS:
columns.types string
file.inputformat org.apache.hadoop.mapred.TextInputFormat
file.outputformat org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat
- location file:/mnt/vol/devrs004.snc1/jssarma/projects/hive_trunk/build/ql/test/data/warehouse/dest1
+ location pfile:/data/users/jssarma/hive_trunk/build/ql/test/data/warehouse/dest1
name dest1
serialization.ddl struct dest1 { string mydata}
serialization.format 1
serialization.last.column.takes.rest true
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- transient_lastDdlTime 1279735739
+ transient_lastDdlTime 1280083027
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: dest1
- tmp directory: file:/mnt/vol/devrs004.snc1/jssarma/projects/hive_trunk/build/ql/scratchdir/hive_2010-07-21_11-08-59_521_6439138630072861096/10001
+ tmp directory: pfile:/data/users/jssarma/hive_trunk/build/ql/scratchdir/hive_2010-07-25_11-37-07_143_545511266047909547/-ext-10001
Stage: Stage-2
Map Reduce
Alias -> Map Operator Tree:
- file:/mnt/vol/devrs004.snc1/jssarma/projects/hive_trunk/build/ql/scratchdir/hive_2010-07-21_11-08-59_521_6439138630072861096/10002
+ pfile:/data/users/jssarma/hive_trunk/build/ql/scratchdir/hive_2010-07-25_11-37-07_143_545511266047909547/-ext-10002
Reduce Output Operator
sort order:
Map-reduce partition columns:
@@ -201,11 +201,11 @@ STAGE PLANS:
type: string
Needs Tagging: false
Path -> Alias:
- file:/mnt/vol/devrs004.snc1/jssarma/projects/hive_trunk/build/ql/scratchdir/hive_2010-07-21_11-08-59_521_6439138630072861096/10002 [file:/mnt/vol/devrs004.snc1/jssarma/projects/hive_trunk/build/ql/scratchdir/hive_2010-07-21_11-08-59_521_6439138630072861096/10002]
+ pfile:/data/users/jssarma/hive_trunk/build/ql/scratchdir/hive_2010-07-25_11-37-07_143_545511266047909547/-ext-10002 [pfile:/data/users/jssarma/hive_trunk/build/ql/scratchdir/hive_2010-07-25_11-37-07_143_545511266047909547/-ext-10002]
Path -> Partition:
- file:/mnt/vol/devrs004.snc1/jssarma/projects/hive_trunk/build/ql/scratchdir/hive_2010-07-21_11-08-59_521_6439138630072861096/10002
+ pfile:/data/users/jssarma/hive_trunk/build/ql/scratchdir/hive_2010-07-25_11-37-07_143_545511266047909547/-ext-10002
Partition
- base file name: 10002
+ base file name: -ext-10002
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat
properties:
@@ -214,13 +214,13 @@ STAGE PLANS:
columns.types string
file.inputformat org.apache.hadoop.mapred.TextInputFormat
file.outputformat org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat
- location file:/mnt/vol/devrs004.snc1/jssarma/projects/hive_trunk/build/ql/test/data/warehouse/dest1
+ location pfile:/data/users/jssarma/hive_trunk/build/ql/test/data/warehouse/dest1
name dest1
serialization.ddl struct dest1 { string mydata}
serialization.format 1
serialization.last.column.takes.rest true
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- transient_lastDdlTime 1279735739
+ transient_lastDdlTime 1280083027
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
input format: org.apache.hadoop.mapred.TextInputFormat
@@ -231,13 +231,13 @@ STAGE PLANS:
columns.types string
file.inputformat org.apache.hadoop.mapred.TextInputFormat
file.outputformat org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat
- location file:/mnt/vol/devrs004.snc1/jssarma/projects/hive_trunk/build/ql/test/data/warehouse/dest1
+ location pfile:/data/users/jssarma/hive_trunk/build/ql/test/data/warehouse/dest1
name dest1
serialization.ddl struct dest1 { string mydata}
serialization.format 1
serialization.last.column.takes.rest true
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- transient_lastDdlTime 1279735739
+ transient_lastDdlTime 1280083027
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: dest1
name: dest1
@@ -246,7 +246,7 @@ STAGE PLANS:
File Output Operator
compressed: false
GlobalTableId: 0
- directory: file:/mnt/vol/devrs004.snc1/jssarma/projects/hive_trunk/build/ql/scratchdir/hive_2010-07-21_11-08-59_521_6439138630072861096/10000
+ directory: pfile:/data/users/jssarma/hive_trunk/build/ql/scratchdir/hive_2010-07-25_11-37-07_143_545511266047909547/-ext-10000
NumFilesPerFileSink: 1
table:
input format: org.apache.hadoop.mapred.TextInputFormat
@@ -257,13 +257,13 @@ STAGE PLANS:
columns.types string
file.inputformat org.apache.hadoop.mapred.TextInputFormat
file.outputformat org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat
- location file:/mnt/vol/devrs004.snc1/jssarma/projects/hive_trunk/build/ql/test/data/warehouse/dest1
+ location pfile:/data/users/jssarma/hive_trunk/build/ql/test/data/warehouse/dest1
name dest1
serialization.ddl struct dest1 { string mydata}
serialization.format 1
serialization.last.column.takes.rest true
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- transient_lastDdlTime 1279735739
+ transient_lastDdlTime 1280083027
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: dest1
TotalFiles: 1
@@ -303,12 +303,12 @@ PREHOOK: query: -- Test the result
SELECT * FROM dest1
PREHOOK: type: QUERY
PREHOOK: Input: default@dest1
-PREHOOK: Output: file:/tmp/jssarma/hive_2010-07-21_11-09-02_211_5180340221675233412/10000
+PREHOOK: Output: file:/tmp/jssarma/hive_2010-07-25_11-37-10_119_5660806743258606681/-mr-10000
POSTHOOK: query: -- Test the result
SELECT * FROM dest1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@dest1
-POSTHOOK: Output: file:/tmp/jssarma/hive_2010-07-21_11-09-02_211_5180340221675233412/10000
+POSTHOOK: Output: file:/tmp/jssarma/hive_2010-07-25_11-37-10_119_5660806743258606681/-mr-10000
POSTHOOK: Lineage: dest1.mydata SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
238 val_238
86 val_86