You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by am...@apache.org on 2015/08/10 14:21:38 UTC
hive git commit: HIVE-11376 : Removes legacy code wrt skipping files
with codec to HiveInputFormat from CombineHiveInputFormat (Rajat Khandelwal,
reviewed by Amareshwari)
Repository: hive
Updated Branches:
refs/heads/master 6df52edc5 -> cfda5700a
HIVE-11376 : Removes legacy code wrt skipping files with codec to HiveInputFormat from CombineHiveInputFormat (Rajat Khandelwal, reviewed by Amareshwari)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cfda5700
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cfda5700
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cfda5700
Branch: refs/heads/master
Commit: cfda5700a715b71f5c4c6b325d4adaa213ff7618
Parents: 6df52ed
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Mon Aug 10 17:51:09 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Mon Aug 10 17:51:09 2015 +0530
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 2 -
.../hive/ql/io/CombineHiveInputFormat.java | 39 --------------------
.../hive/ql/optimizer/GenMapRedUtils.java | 2 -
.../org/apache/hadoop/hive/ql/plan/MapWork.java | 10 -----
.../apache/hadoop/hive/ql/plan/PlanUtils.java | 9 +----
5 files changed, 1 insertion(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/cfda5700/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 36bb394..9cc7987 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1175,8 +1175,6 @@ public class HiveConf extends Configuration {
HIVEROWOFFSET("hive.exec.rowoffset", false,
"Whether to provide the row offset virtual column"),
- HIVE_COMBINE_INPUT_FORMAT_SUPPORTS_SPLITTABLE("hive.hadoop.supports.splittable.combineinputformat", false, ""),
-
// Optimizer
HIVEOPTINDEXFILTER("hive.optimize.index.filter", false,
"Whether to enable automatic use of indexes"),
http://git-wip-us.apache.org/repos/asf/hive/blob/cfda5700/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
index e13c4dd..11740d1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
@@ -374,45 +374,6 @@ public class CombineHiveInputFormat<K extends WritableComparable, V extends Writ
}
FileSystem inpFs = path.getFileSystem(job);
- // Since there is no easy way of knowing whether MAPREDUCE-1597 is present in the tree or not,
- // we use a configuration variable for the same
- if (this.mrwork != null && !this.mrwork.getHadoopSupportsSplittable()) {
- // The following code should be removed, once
- // https://issues.apache.org/jira/browse/MAPREDUCE-1597 is fixed.
- // Hadoop does not handle non-splittable files correctly for CombineFileInputFormat,
- // so don't use CombineFileInputFormat for non-splittable files
-
- //ie, dont't combine if inputformat is a TextInputFormat and has compression turned on
-
- if (inputFormat instanceof TextInputFormat) {
- Queue<Path> dirs = new LinkedList<Path>();
- FileStatus fStats = inpFs.getFileStatus(path);
-
- // If path is a directory
- if (fStats.isDir()) {
- dirs.offer(path);
- } else if ((new CompressionCodecFactory(job)).getCodec(path) != null) {
- //if compresssion codec is set, use HiveInputFormat.getSplits (don't combine)
- splits = super.getSplits(job, numSplits);
- return splits;
- }
-
- while (dirs.peek() != null) {
- Path tstPath = dirs.remove();
- FileStatus[] fStatus = inpFs.listStatus(tstPath, FileUtils.HIDDEN_FILES_PATH_FILTER);
- for (int idx = 0; idx < fStatus.length; idx++) {
- if (fStatus[idx].isDir()) {
- dirs.offer(fStatus[idx].getPath());
- } else if ((new CompressionCodecFactory(job)).getCodec(
- fStatus[idx].getPath()) != null) {
- //if compresssion codec is set, use HiveInputFormat.getSplits (don't combine)
- splits = super.getSplits(job, numSplits);
- return splits;
- }
- }
- }
- }
- }
//don't combine if inputformat is a SymlinkTextInputFormat
if (inputFormat instanceof SymlinkTextInputFormat) {
splits = super.getSplits(job, numSplits);
http://git-wip-us.apache.org/repos/asf/hive/blob/cfda5700/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
index 693d8c7..4a325fb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
@@ -933,8 +933,6 @@ public final class GenMapRedUtils {
work.setPathToAliases(new LinkedHashMap<String, ArrayList<String>>());
work.setPathToPartitionInfo(new LinkedHashMap<String, PartitionDesc>());
work.setAliasToWork(new LinkedHashMap<String, Operator<? extends OperatorDesc>>());
- work.setHadoopSupportsSplittable(
- conf.getBoolVar(HiveConf.ConfVars.HIVE_COMBINE_INPUT_FORMAT_SUPPORTS_SPLITTABLE));
return mrWork;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/cfda5700/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
index 2cb9257..bc9b645 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
@@ -63,8 +63,6 @@ public class MapWork extends BaseWork {
private static final Log LOG = LogFactory.getLog(MapWork.class);
- private boolean hadoopSupportsSplittable;
-
// use LinkedHashMap to make sure the iteration order is
// deterministic, to ease testing
private LinkedHashMap<String, ArrayList<String>> pathToAliases = new LinkedHashMap<String, ArrayList<String>>();
@@ -421,14 +419,6 @@ public class MapWork extends BaseWork {
return this.mapperCannotSpanPartns;
}
- public boolean getHadoopSupportsSplittable() {
- return hadoopSupportsSplittable;
- }
-
- public void setHadoopSupportsSplittable(boolean hadoopSupportsSplittable) {
- this.hadoopSupportsSplittable = hadoopSupportsSplittable;
- }
-
public String getIndexIntermediateFile() {
return indexIntermediateFile;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/cfda5700/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
index 76926e7..b50eaab 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
@@ -99,14 +99,7 @@ public final class PlanUtils {
@SuppressWarnings("nls")
public static MapredWork getMapRedWork() {
- try {
- MapredWork work = new MapredWork();
- work.getMapWork().setHadoopSupportsSplittable(Hive.get().getConf().getBoolVar(
- HiveConf.ConfVars.HIVE_COMBINE_INPUT_FORMAT_SUPPORTS_SPLITTABLE));
- return work;
- } catch (HiveException ex) {
- throw new RuntimeException(ex);
- }
+ return new MapredWork();
}
public static TableDesc getDefaultTableDesc(CreateTableDesc directoryDesc,