You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/06/04 05:14:18 UTC
svn commit: r1489279 - in /hive/trunk/ql/src/java/org/apache/hadoop/hive/ql:
exec/ index/ io/rcfile/merge/ io/rcfile/stats/ io/rcfile/truncate/ parse/
Author: hashutosh
Date: Tue Jun 4 03:14:17 2013
New Revision: 1489279
URL: http://svn.apache.org/r1489279
Log:
HIVE-4585 : Remove unused MR Temp file localization from Tasks (Gunther Hagleitner via Ashutosh Chauhan)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ConditionalTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DependencyCollectionTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java Tue Jun 4 03:14:17 2013
@@ -390,17 +390,4 @@ public class ColumnStatsTask extends Tas
public String getName() {
return "COLUMNSTATS TASK";
}
-
- @Override
- protected void localizeMRTmpFilesImpl(Context ctx) {
- FetchWork fWork = work.getfWork();
- String s = fWork.getTblDir();
- if ((s != null) && ctx.isMRTmpFileURI(s)) {
- fWork.setTblDir(ctx.localizeMRTmpFileURI(s));
- }
- ArrayList<String> ls = fWork.getPartDir();
- if (ls != null) {
- ctx.localizePaths(ls);
- }
- }
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ConditionalTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ConditionalTask.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ConditionalTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ConditionalTask.java Tue Jun 4 03:14:17 2013
@@ -205,15 +205,6 @@ public class ConditionalTask extends Tas
}
@Override
- protected void localizeMRTmpFilesImpl(Context ctx) {
- if (getListTasks() != null) {
- for (Task<? extends Serializable> t : getListTasks()) {
- t.localizeMRTmpFiles(ctx);
- }
- }
- }
-
- @Override
public List<Task<? extends Serializable>> getDependentTasks() {
return listTasks;
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java Tue Jun 4 03:14:17 2013
@@ -106,12 +106,4 @@ public class CopyTask extends Task<CopyW
public String getName() {
return "COPY";
}
-
- @Override
- protected void localizeMRTmpFilesImpl(Context ctx) {
- // copy task is only used by the load command and
- // does not use any map-reduce tmp files
- // we don't expect to enter this code path at all
- throw new RuntimeException ("Unexpected call");
- }
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java Tue Jun 4 03:14:17 2013
@@ -4059,9 +4059,4 @@ public class DDLTask extends Task<DDLWor
public String getName() {
return "DDL";
}
-
- @Override
- protected void localizeMRTmpFilesImpl(Context ctx) {
- // no-op
- }
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DependencyCollectionTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DependencyCollectionTask.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DependencyCollectionTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DependencyCollectionTask.java Tue Jun 4 03:14:17 2013
@@ -53,12 +53,4 @@ public class DependencyCollectionTask ex
public String getName() {
return "DEPENDENCY_COLLECTION";
}
-
- @Override
- protected void localizeMRTmpFilesImpl(Context ctx) {
- // copy task doesn't have any execution and so
- // does not use any map-reduce tmp files
- // we don't expect to enter this code path at all
- throw new RuntimeException ("Unexpected call");
- }
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java Tue Jun 4 03:14:17 2013
@@ -915,57 +915,6 @@ public class ExecDriver extends Task<Map
}
@Override
- protected void localizeMRTmpFilesImpl(Context ctx) {
-
- // localize any map-reduce input paths
- ctx.localizeKeys((Map<String, Object>) ((Object) work.getPathToAliases()));
- ctx.localizeKeys((Map<String, Object>) ((Object) work.getPathToPartitionInfo()));
-
- // localize any input paths for maplocal work
- MapredLocalWork l = work.getMapLocalWork();
- if (l != null) {
- Map<String, FetchWork> m = l.getAliasToFetchWork();
- if (m != null) {
- for (FetchWork fw : m.values()) {
- String s = fw.getTblDir();
- if ((s != null) && ctx.isMRTmpFileURI(s)) {
- fw.setTblDir(ctx.localizeMRTmpFileURI(s));
- }
- }
- }
- }
-
- // fix up outputs
- Map<String, ArrayList<String>> pa = work.getPathToAliases();
- if (pa != null) {
- for (List<String> ls : pa.values()) {
- for (String a : ls) {
- ArrayList<Operator<? extends OperatorDesc>> opList =
- new ArrayList<Operator<? extends OperatorDesc>>();
- opList.add(work.getAliasToWork().get(a));
-
- while (!opList.isEmpty()) {
- Operator<? extends OperatorDesc> op = opList.remove(0);
-
- if (op instanceof FileSinkOperator) {
- FileSinkDesc fdesc = ((FileSinkOperator) op).getConf();
- String s = fdesc.getDirName();
- if ((s != null) && ctx.isMRTmpFileURI(s)) {
- fdesc.setDirName(ctx.localizeMRTmpFileURI(s));
- }
- ((FileSinkOperator) op).setConf(fdesc);
- }
-
- if (op.getChildOperators() != null) {
- opList.addAll(op.getChildOperators());
- }
- }
- }
- }
- }
- }
-
- @Override
public void updateCounters(Counters ctrs, RunningJob rj) throws IOException {
for (Operator<? extends OperatorDesc> op : work.getAliasToWork().values()) {
op.updateCounters(ctrs);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java Tue Jun 4 03:14:17 2013
@@ -717,13 +717,6 @@ public class ExplainTask extends Task<Ex
}
@Override
- protected void localizeMRTmpFilesImpl(Context ctx) {
- // explain task has nothing to localize
- // we don't expect to enter this code path at all
- throw new RuntimeException("Unexpected call");
- }
-
- @Override
public List<FieldSchema> getResultSchema() {
FieldSchema tmpFieldSchema = new FieldSchema();
List<FieldSchema> colList = new ArrayList<FieldSchema>();
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java Tue Jun 4 03:14:17 2013
@@ -163,19 +163,6 @@ public class FetchTask extends Task<Fetc
return "FETCH";
}
- @Override
- protected void localizeMRTmpFilesImpl(Context ctx) {
- String s = work.getTblDir();
- if ((s != null) && ctx.isMRTmpFileURI(s)) {
- work.setTblDir(ctx.localizeMRTmpFileURI(s));
- }
-
- ArrayList<String> ls = work.getPartDir();
- if (ls != null) {
- ctx.localizePaths(ls);
- }
- }
-
/**
* Clear the Fetch Operator.
*
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java Tue Jun 4 03:14:17 2013
@@ -114,9 +114,4 @@ public class FunctionTask extends Task<F
public String getName() {
return "FUNCTION";
}
-
- @Override
- protected void localizeMRTmpFilesImpl(Context ctx) {
- throw new RuntimeException ("Unexpected call");
- }
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java Tue Jun 4 03:14:17 2013
@@ -448,11 +448,6 @@ public class MapredLocalTask extends Tas
}
@Override
- public void localizeMRTmpFilesImpl(Context ctx) {
-
- }
-
- @Override
public boolean isMapRedLocalTask() {
return true;
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java Tue Jun 4 03:14:17 2013
@@ -537,10 +537,4 @@ public class MoveTask extends Task<MoveW
public String getName() {
return "MOVE";
}
-
-
- @Override
- protected void localizeMRTmpFilesImpl(Context ctx) {
- // no-op
- }
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java Tue Jun 4 03:14:17 2013
@@ -263,11 +263,6 @@ public class StatsTask extends Task<Stat
return "STATS";
}
- @Override
- protected void localizeMRTmpFilesImpl(Context ctx) {
- // Nothing to do for StatsTask here.
- }
-
private int aggregateStats() {
StatsAggregator statsAggregator = null;
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java Tue Jun 4 03:14:17 2013
@@ -384,37 +384,6 @@ public abstract class Task<T extends Ser
public abstract StageType getType();
/**
- * If this task uses any map-reduce intermediate data (either for reading or for writing),
- * localize them (using the supplied Context). Map-Reduce intermediate directories are allocated
- * using Context.getMRTmpFileURI() and can be localized using localizeMRTmpFileURI().
- *
- * This method is declared abstract to force any task code to explicitly deal with this aspect of
- * execution.
- *
- * @param ctx
- * context object with which to localize
- */
- abstract protected void localizeMRTmpFilesImpl(Context ctx);
-
- /**
- * Localize a task tree
- *
- * @param ctx
- * context object with which to localize
- */
- public final void localizeMRTmpFiles(Context ctx) {
- localizeMRTmpFilesImpl(ctx);
-
- if (childTasks == null) {
- return;
- }
-
- for (Task<? extends Serializable> t : childTasks) {
- t.localizeMRTmpFiles(ctx);
- }
- }
-
- /**
* Subscribe the feed of publisher. To prevent cycles, a task can only subscribe to its ancestor.
* Feed is a generic form of execution-time feedback (type, value) pair from one task to another
* task. Examples include dynamic partitions (which are only available at execution time). The
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java Tue Jun 4 03:14:17 2013
@@ -98,9 +98,4 @@ public class IndexMetadataChangeTask ext
public StageType getType() {
return StageType.DDL;
}
-
- @Override
- protected void localizeMRTmpFilesImpl(Context ctx) {
- }
-
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java Tue Jun 4 03:14:17 2013
@@ -378,10 +378,4 @@ public class BlockMergeTask extends Task
public void updateCounters(Counters ctrs, RunningJob rj) throws IOException {
// no op
}
-
- @Override
- protected void localizeMRTmpFilesImpl(Context ctx) {
- // no op
- }
-
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java Tue Jun 4 03:14:17 2013
@@ -379,11 +379,4 @@ public class PartialScanTask extends Tas
public void updateCounters(Counters ctrs, RunningJob rj) throws IOException {
// no op
}
-
- @Override
- protected void localizeMRTmpFilesImpl(Context ctx) {
- // no op
- }
-
-
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java Tue Jun 4 03:14:17 2013
@@ -254,10 +254,4 @@ public class ColumnTruncateTask extends
public void updateCounters(Counters ctrs, RunningJob rj) throws IOException {
// no op
}
-
- @Override
- protected void localizeMRTmpFilesImpl(Context ctx) {
- // no op
- }
-
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Tue Jun 4 03:14:17 2013
@@ -9637,15 +9637,6 @@ public class SemanticAnalyzer extends Ba
ctx.setOriginalTracker(ShimLoader.getHadoopShims().getJobLauncherRpcAddress(conf));
ShimLoader.getHadoopShims().setJobLauncherRpcAddress(conf, "local");
console.printInfo("Automatically selecting local only mode for query");
-
- // If all the tasks can be run locally, we can use local disk for
- // storing intermediate data.
-
- /**
- * This code is commented out pending further testing/development
- * for (Task<? extends OperatorDesc> t: rootTasks)
- * t.localizeMRTmpFiles(ctx);
- */
}
}