You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sz...@apache.org on 2015/01/09 02:53:14 UTC
svn commit: r1650429 - in /hive/branches/spark:
ql/src/java/org/apache/hadoop/hive/ql/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/
ql/src/java/org/apache/hadoop/hive/ql/io/ ql/src/java/org/apache/...
Author: szehon
Date: Fri Jan 9 01:53:13 2015
New Revision: 1650429
URL: http://svn.apache.org/r1650429
Log:
HIVE-9319 : Cleanup Modified Files [Spark Branch] (Szehon, reviewed by Xuefu)
Modified:
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainerSerDe.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/io/HiveKey.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcessor.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
hive/branches/spark/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/Driver.java?rev=1650429&r1=1650428&r2=1650429&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/Driver.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/Driver.java Fri Jan 9 01:53:13 2015
@@ -1342,9 +1342,9 @@ public class Driver implements CommandPr
}
}
- int jobs = Utilities.getMRTasks(plan.getRootTasks()).size() +
- Utilities.getTezTasks(plan.getRootTasks()).size() +
- Utilities.getSparkTasks(plan.getRootTasks()).size();
+ int jobs = Utilities.getMRTasks(plan.getRootTasks()).size()
+ + Utilities.getTezTasks(plan.getRootTasks()).size()
+ + Utilities.getSparkTasks(plan.getRootTasks()).size();
if (jobs > 0) {
console.printInfo("Query ID = " + plan.getQueryId());
console.printInfo("Total jobs = " + jobs);
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java?rev=1650429&r1=1650428&r2=1650429&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java Fri Jan 9 01:53:13 2015
@@ -424,12 +424,12 @@ public class ExplainTask extends Task<Ex
json.accumulate(ent.getKey().toString(), jsonDep);
}
}
- } else if (ent.getValue() != null && !((List<?>)ent.getValue()).isEmpty()
- && ((List<?>)ent.getValue()).get(0) != null &&
- ((List<?>)ent.getValue()).get(0) instanceof SparkWork.Dependency) {
+ } else if (ent.getValue() != null && !((List<?>) ent.getValue()).isEmpty()
+ && ((List<?>) ent.getValue()).get(0) != null &&
+ ((List<?>) ent.getValue()).get(0) instanceof SparkWork.Dependency) {
if (out != null) {
boolean isFirst = true;
- for (SparkWork.Dependency dep: (List<SparkWork.Dependency>)ent.getValue()) {
+ for (SparkWork.Dependency dep: (List<SparkWork.Dependency>) ent.getValue()) {
if (!isFirst) {
out.print(", ");
} else {
@@ -446,7 +446,7 @@ public class ExplainTask extends Task<Ex
out.println();
}
if (jsonOutput) {
- for (SparkWork.Dependency dep: (List<SparkWork.Dependency>)ent.getValue()) {
+ for (SparkWork.Dependency dep: (List<SparkWork.Dependency>) ent.getValue()) {
JSONObject jsonDep = new JSONObject();
jsonDep.put("parent", dep.getName());
jsonDep.put("type", dep.getShuffleType());
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1650429&r1=1650428&r2=1650429&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Fri Jan 9 01:53:13 2015
@@ -363,8 +363,8 @@ public final class Utilities {
path = getPlanPath(conf, name);
LOG.info("PLAN PATH = " + path);
assert path != null;
- if (!gWorkMap.containsKey(path) ||
- HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("spark")) {
+ if (!gWorkMap.containsKey(path)
+ || HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("spark")) {
Path localPath;
if (conf.getBoolean("mapreduce.task.uberized", false) && name.equals(REDUCE_PLAN_NAME)) {
localPath = new Path(name);
@@ -2673,7 +2673,8 @@ public final class Utilities {
return sparkTasks;
}
- private static void getSparkTasks(List<Task<? extends Serializable>> tasks, List<SparkTask> sparkTasks) {
+ private static void getSparkTasks(List<Task<? extends Serializable>> tasks,
+ List<SparkTask> sparkTasks) {
for (Task<? extends Serializable> task : tasks) {
if (task instanceof SparkTask && !sparkTasks.contains(task)) {
sparkTasks.add((SparkTask) task);
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainerSerDe.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainerSerDe.java?rev=1650429&r1=1650428&r2=1650429&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainerSerDe.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainerSerDe.java Fri Jan 9 01:53:13 2015
@@ -154,7 +154,7 @@ public class MapJoinTableContainerSerDe
return tableContainer;
} catch (IOException e) {
throw new HiveException("IO error while trying to create table container", e);
- } catch(Exception e) {
+ } catch (Exception e) {
throw new HiveException("Error while trying to create table container", e);
}
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/io/HiveKey.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/io/HiveKey.java?rev=1650429&r1=1650428&r2=1650429&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/io/HiveKey.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/io/HiveKey.java Fri Jan 9 01:53:13 2015
@@ -24,7 +24,6 @@ import org.apache.hadoop.io.WritableComp
/**
* HiveKey is a simple wrapper on Text which allows us to set the hashCode
* easily. hashCode is used for hadoop partitioner.
- *
*/
public class HiveKey extends BytesWritable {
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java?rev=1650429&r1=1650428&r2=1650429&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java Fri Jan 9 01:53:13 2015
@@ -861,10 +861,10 @@ public final class GenMapRedUtils {
}
}
} else if (task instanceof SparkTask) {
- SparkWork work = (SparkWork)task.getWork();
+ SparkWork work = (SparkWork) task.getWork();
for (BaseWork w : work.getAllWorkUnsorted()) {
if (w instanceof MapWork) {
- ((MapWork)w).deriveExplainAttributes();
+ ((MapWork) w).deriveExplainAttributes();
}
}
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java?rev=1650429&r1=1650428&r2=1650429&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java Fri Jan 9 01:53:13 2015
@@ -102,10 +102,12 @@ public class Optimizer {
}
transformations.add(new SamplePruner());
- MapJoinProcessor mapJoinProcessor = isSparkExecEngine ? new SparkMapJoinProcessor() : new MapJoinProcessor();
+ MapJoinProcessor mapJoinProcessor = isSparkExecEngine ? new SparkMapJoinProcessor()
+ : new MapJoinProcessor();
transformations.add(mapJoinProcessor);
- if ((HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTBUCKETMAPJOIN)) && !isTezExecEngine && !isSparkExecEngine) {
+ if ((HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTBUCKETMAPJOIN))
+ && !isTezExecEngine && !isSparkExecEngine) {
transformations.add(new BucketMapJoinOptimizer());
bucketMapJoinOptimizer = true;
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcessor.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcessor.java?rev=1650429&r1=1650428&r2=1650429&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcessor.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcessor.java Fri Jan 9 01:53:13 2015
@@ -96,8 +96,8 @@ public class UnionProcessor implements T
// Walk the tree again to see if the union can be removed completely
HiveConf conf = pCtx.getConf();
opRules.clear();
- if (conf.getBoolVar(HiveConf.ConfVars.HIVE_OPTIMIZE_UNION_REMOVE) &&
- !conf.getVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("spark")) {
+ if (conf.getBoolVar(HiveConf.ConfVars.HIVE_OPTIMIZE_UNION_REMOVE)
+ && !conf.getVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("spark")) {
if (!conf.getBoolVar(HiveConf.ConfVars.HIVE_HADOOP_SUPPORTS_SUBDIRECTORIES)) {
throw new
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java?rev=1650429&r1=1650428&r2=1650429&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java Fri Jan 9 01:53:13 2015
@@ -147,9 +147,7 @@ public class MapReduceCompiler extends T
}
// loop over all the operators recursively
- // TODO: changed from private to protected for SparkCompiler to use. It will be changed back onece SparkCompiler
- // stands alone.
- protected void breakOperatorTree(Operator<? extends OperatorDesc> topOp) {
+ private void breakOperatorTree(Operator<? extends OperatorDesc> topOp) {
if (topOp instanceof ReduceSinkOperator) {
topOp.setChildOperators(null);
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java?rev=1650429&r1=1650428&r2=1650429&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java Fri Jan 9 01:53:13 2015
@@ -129,7 +129,7 @@ public abstract class BaseWork extends A
// add all children
opStack.addAll(opSet);
- while(!opStack.empty()) {
+ while (!opStack.empty()) {
Operator<?> op = opStack.pop();
if (op.getNumChild() == 0) {
returnSet.add(op);
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java?rev=1650429&r1=1650428&r2=1650429&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java Fri Jan 9 01:53:13 2015
@@ -149,7 +149,7 @@ public class ConditionalResolverMergeFil
work = ((MapredWork) mrTask.getWork()).getMapWork();
} else if (mrTask.getWork() instanceof TezWork){
work = (MapWork) ((TezWork) mrTask.getWork()).getAllWork().get(0);
- } else if (mrTask.getWork() instanceof SparkWork){
+ } else if (mrTask.getWork() instanceof SparkWork) {
work = (MapWork) ((SparkWork) mrTask.getWork()).getAllWork().get(0);
} else {
work = (MapWork) mrTask.getWork();
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java?rev=1650429&r1=1650428&r2=1650429&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java Fri Jan 9 01:53:13 2015
@@ -1271,7 +1271,7 @@ public class SessionState {
if (sparkSession != null) {
try {
SparkSessionManagerImpl.getInstance().closeSession(sparkSession);
- } catch(Exception ex) {
+ } catch (Exception ex) {
LOG.error("Error closing spark session.", ex);
} finally {
sparkSession = null;
Modified: hive/branches/spark/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java?rev=1650429&r1=1650428&r2=1650429&view=diff
==============================================================================
--- hive/branches/spark/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java (original)
+++ hive/branches/spark/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java Fri Jan 9 01:53:13 2015
@@ -518,14 +518,14 @@ public interface HadoopShims {
Path getCurrentTrashPath(Configuration conf, FileSystem fs);
/**
- * Returns a shim to wrap KerberosName
+ * Check whether file is directory.
*/
- public KerberosNameShim getKerberosNameShim(String name) throws IOException;
+ boolean isDirectory(FileStatus fileStatus);
/**
- * Check whether file is directory.
+ * Returns a shim to wrap KerberosName
*/
- boolean isDirectory(FileStatus fileStatus);
+ public KerberosNameShim getKerberosNameShim(String name) throws IOException;
/**
* Shim for KerberosName