You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jx...@apache.org on 2015/10/28 16:06:40 UTC
[1/2] hive git commit: HIVE-12268: Context leaks deleteOnExit paths
(Jimmy, reviewed by Szehon)
Repository: hive
Updated Branches:
refs/heads/master efbb0418a -> 34ba81ae7
HIVE-12268: Context leaks deleteOnExit paths (Jimmy, reviewed by Szehon)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/34ba81ae
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/34ba81ae
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/34ba81ae
Branch: refs/heads/master
Commit: 34ba81ae774bc4e13e7b3f8c1454490fb8d02704
Parents: 9c18fce
Author: Jimmy Xiang <jx...@cloudera.com>
Authored: Mon Oct 26 16:24:48 2015 -0700
Committer: Jimmy Xiang <jx...@cloudera.com>
Committed: Wed Oct 28 07:55:21 2015 -0700
----------------------------------------------------------------------
ql/src/java/org/apache/hadoop/hive/ql/Context.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/34ba81ae/ql/src/java/org/apache/hadoop/hive/ql/Context.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Context.java b/ql/src/java/org/apache/hadoop/hive/ql/Context.java
index ca0d487..1499a91 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Context.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Context.java
@@ -347,7 +347,9 @@ public class Context {
for (Map.Entry<String, Path> entry : fsScratchDirs.entrySet()) {
try {
Path p = entry.getValue();
- p.getFileSystem(conf).delete(p, true);
+ FileSystem fs = p.getFileSystem(conf);
+ fs.delete(p, true);
+ fs.cancelDeleteOnExit(p);
} catch (Exception e) {
LOG.warn("Error Removing Scratch: "
+ StringUtils.stringifyException(e));
[2/2] hive git commit: HIVE-12265: Generate lineage info only if
requested (Jimmy, reviewed by Chao)
Posted by jx...@apache.org.
HIVE-12265: Generate lineage info only if requested (Jimmy, reviewed by Chao)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9c18fce4
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9c18fce4
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9c18fce4
Branch: refs/heads/master
Commit: 9c18fce4bfd60288250a90797ce35d042bd91ec7
Parents: efbb041
Author: Jimmy Xiang <jx...@cloudera.com>
Authored: Mon Oct 26 10:20:25 2015 -0700
Committer: Jimmy Xiang <jx...@cloudera.com>
Committed: Wed Oct 28 07:55:21 2015 -0700
----------------------------------------------------------------------
ql/src/java/org/apache/hadoop/hive/ql/Driver.java | 9 +++++----
.../org/apache/hadoop/hive/ql/optimizer/Optimizer.java | 13 ++++++++++++-
2 files changed, 17 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/9c18fce4/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index 3a3fcf1..44b247f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -298,9 +298,7 @@ public class Driver implements CommandProcessor {
}
public Driver() {
- conf = (SessionState.get() != null) ? SessionState.get().getConf() : null;
- isParallelEnabled = (conf != null)
- && HiveConf.getBoolVar(conf, ConfVars.HIVE_SERVER2_PARALLEL_COMPILATION);
+ this((SessionState.get() != null) ? SessionState.get().getConf() : null);
}
/**
@@ -1085,6 +1083,10 @@ public class Driver implements CommandProcessor {
* while keeping the result around.
*/
private void releaseResources() {
+ if (SessionState.get() != null) {
+ SessionState.get().getLineageState().clear();
+ }
+
if (plan != null) {
fetchTask = plan.getFetchTask();
if (fetchTask != null) {
@@ -1713,7 +1715,6 @@ public class Driver implements CommandProcessor {
if (SessionState.get() != null) {
try {
- SessionState.get().getLineageState().clear();
SessionState.get().getHiveHistory().logPlanProgress(plan);
} catch (Exception e) {
// ignore
http://git-wip-us.apache.org/repos/asf/hive/blob/9c18fce4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
index 439f616..25c9618 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.optimizer;
import java.util.ArrayList;
import java.util.List;
+import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -41,6 +42,10 @@ import org.apache.hadoop.hive.ql.ppd.PredicatePushDown;
import org.apache.hadoop.hive.ql.ppd.PredicateTransitivePropagate;
import org.apache.hadoop.hive.ql.ppd.SyntheticJoinPredicate;
+import com.google.common.base.Splitter;
+import com.google.common.base.Strings;
+import com.google.common.collect.Sets;
+
/**
* Implementation of the optimizer.
*/
@@ -67,7 +72,13 @@ public class Optimizer {
transformations.add(new HiveOpConverterPostProc());
// Add the transformation that computes the lineage information.
- transformations.add(new Generator());
+ Set<String> postExecHooks = Sets.newHashSet(
+ Splitter.on(",").trimResults().omitEmptyStrings().split(
+ Strings.nullToEmpty(HiveConf.getVar(hiveConf, HiveConf.ConfVars.POSTEXECHOOKS))));
+ if (postExecHooks.contains("org.apache.hadoop.hive.ql.hooks.PostExecutePrinter")
+ || postExecHooks.contains("org.apache.hadoop.hive.ql.hooks.LineageLogger")) {
+ transformations.add(new Generator());
+ }
// Try to transform OR predicates in Filter into simpler IN clauses first
if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEPOINTLOOKUPOPTIMIZER)) {