You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2015/11/23 15:31:57 UTC
hive git commit: HIVE-12461: Branch-1 -Phadoop-1 build is broken
(Aleksei via Xuefu)
Repository: hive
Updated Branches:
refs/heads/branch-1 d640cbcd8 -> 37c8998c5
HIVE-12461: Branch-1 -Phadoop-1 build is broken (Aleksei via Xuefu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/37c8998c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/37c8998c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/37c8998c
Branch: refs/heads/branch-1
Commit: 37c8998c518ce7e76f3240ecaf5126dc1a36af61
Parents: d640cbc
Author: Xuefu Zhang <xz...@Cloudera.com>
Authored: Mon Nov 23 06:31:45 2015 -0800
Committer: Xuefu Zhang <xz...@Cloudera.com>
Committed: Mon Nov 23 06:31:45 2015 -0800
----------------------------------------------------------------------
common/src/java/org/apache/hadoop/hive/conf/HiveConf.java | 2 ++
ql/src/java/org/apache/hadoop/hive/ql/Context.java | 4 +---
.../main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java | 6 ++++++
.../main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java | 6 ++++++
.../main/java/org/apache/hadoop/hive/shims/HadoopShims.java | 5 +++++
5 files changed, 20 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/37c8998c/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 482bd79..eea1e7c 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -770,6 +770,8 @@ public class HiveConf extends Configuration {
HIVEMAPJOINUSEOPTIMIZEDTABLE("hive.mapjoin.optimized.hashtable", true,
"Whether Hive should use memory-optimized hash table for MapJoin.\n" +
"Only works on Tez and Spark, because memory-optimized hashtable cannot be serialized."),
+ HIVEMAPJOINOPTIMIZEDTABLEPROBEPERCENT("hive.mapjoin.optimized.hashtable.probe.percent",
+ (float) 0.5, "Probing space percentage of the optimized hashtable"),
HIVEUSEHYBRIDGRACEHASHJOIN("hive.mapjoin.hybridgrace.hashtable", true, "Whether to use hybrid" +
"grace hash join as the join method for mapjoin. Tez only."),
HIVEHYBRIDGRACEHASHJOINMEMCHECKFREQ("hive.mapjoin.hybridgrace.memcheckfrequency", 1024, "For " +
http://git-wip-us.apache.org/repos/asf/hive/blob/37c8998c/ql/src/java/org/apache/hadoop/hive/ql/Context.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Context.java b/ql/src/java/org/apache/hadoop/hive/ql/Context.java
index 1499a91..a92331a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Context.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Context.java
@@ -45,7 +45,6 @@ import org.apache.hadoop.hive.ql.exec.TaskRunner;
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.lockmgr.HiveLock;
-import org.apache.hadoop.hive.ql.lockmgr.HiveLockManager;
import org.apache.hadoop.hive.ql.lockmgr.HiveLockObj;
import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager;
import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
@@ -348,8 +347,7 @@ public class Context {
try {
Path p = entry.getValue();
FileSystem fs = p.getFileSystem(conf);
- fs.delete(p, true);
- fs.cancelDeleteOnExit(p);
+ ShimLoader.getHadoopShims().removeAndCancelDeleteOnExit(fs, p);
} catch (Exception e) {
LOG.warn("Error Removing Scratch: "
+ StringUtils.stringifyException(e));
http://git-wip-us.apache.org/repos/asf/hive/blob/37c8998c/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
----------------------------------------------------------------------
diff --git a/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java b/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
index ffffcb7..bf39928 100644
--- a/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
+++ b/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
@@ -722,4 +722,10 @@ public class Hadoop20SShims extends HadoopShimsSecure {
Token<?> fsToken = fs.getDelegationToken(uname);
cred.addToken(fsToken.getService(), fsToken);
}
+
+ @Override
+ public void removeAndCancelDeleteOnExit(FileSystem fs, Path path) throws IOException {
+ fs.delete(path, true);
+ // fs.cancelDeleteOnExit() is not available until 0.23.3
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/37c8998c/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
----------------------------------------------------------------------
diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
index 443153a..7756574 100644
--- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
+++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
@@ -1405,4 +1405,10 @@ public class Hadoop23Shims extends HadoopShimsSecure {
}
return new FastTextReaderShim(in);
}
+
+ @Override
+ public void removeAndCancelDeleteOnExit(FileSystem fs, Path path) throws IOException {
+ fs.delete(path, true);
+ fs.cancelDeleteOnExit(path);
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/37c8998c/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
----------------------------------------------------------------------
diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
index 5c6a4ca..74b7114 100644
--- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
+++ b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
@@ -753,4 +753,9 @@ public interface HadoopShims {
* which are required for TextReaderShim.read() input.
*/
public TextReaderShim getTextReaderShim(InputStream input) throws IOException;
+
+ /**
+ * Delete a specified path and cancel its delete on exit when such method is available.
+ */
+ public void removeAndCancelDeleteOnExit(FileSystem fs, Path path) throws IOException;
}