You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pv...@apache.org on 2020/05/12 10:32:40 UTC
[hive] branch master updated: HIVE-23436: Staging directory is not
removed for stats gathering tasks (Peter Vary reviewed by Zoltan Haindrich)
This is an automated email from the ASF dual-hosted git repository.
pvary pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 8bfdd18 HIVE-23436: Staging directory is not removed for stats gathering tasks (Peter Vary reviewed by Zoltan Haindrich)
8bfdd18 is described below
commit 8bfdd18535eea318cb7709b49e24fc865043eb9f
Author: Peter Vary <pv...@cloudera.com>
AuthorDate: Tue May 12 12:31:56 2020 +0200
HIVE-23436: Staging directory is not removed for stats gathering tasks (Peter Vary reviewed by Zoltan Haindrich)
---
ql/src/java/org/apache/hadoop/hive/ql/Context.java | 16 ++++++++--------
ql/src/java/org/apache/hadoop/hive/ql/Driver.java | 2 +-
.../rebuild/AlterMaterializedViewRebuildAnalyzer.java | 2 +-
.../hive/ql/parse/ColumnStatsAutoGatherContext.java | 1 +
.../hive/ql/parse/ColumnStatsSemanticAnalyzer.java | 4 +---
.../hadoop/hive/ql/parse/LoadSemanticAnalyzer.java | 2 +-
.../hadoop/hive/ql/parse/RewriteSemanticAnalyzer.java | 2 +-
7 files changed, 14 insertions(+), 15 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Context.java b/ql/src/java/org/apache/hadoop/hive/ql/Context.java
index 5e92980..318c207 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Context.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Context.java
@@ -121,7 +121,7 @@ public class Context {
// Some statements, e.g., UPDATE, DELETE, or MERGE, get rewritten into different
// subqueries that create new contexts. We keep them here so we can clean them
// up when we are done.
- private final Set<Context> rewrittenStatementContexts;
+ private final Set<Context> subContexts;
// List of Locks for this query
protected List<HiveLock> hiveLocks;
@@ -325,7 +325,7 @@ public class Context {
private Context(Configuration conf, String executionId) {
this.conf = conf;
this.executionId = executionId;
- this.rewrittenStatementContexts = new HashSet<>();
+ this.subContexts = new HashSet<>();
// local & non-local tmp location is configurable. however it is the same across
// all external file systems
@@ -343,7 +343,7 @@ public class Context {
// hence it needs to be used carefully. In particular, following objects
// are ignored:
// opContext, pathToCS, cboInfo, cboSucceeded, tokenRewriteStream, viewsTokenRewriteStreams,
- // rewrittenStatementContexts, cteTables, loadTableOutputMap, planMapper, insertBranchToNamePrefix
+ // subContexts, cteTables, loadTableOutputMap, planMapper, insertBranchToNamePrefix
this.isHDFSCleanup = ctx.isHDFSCleanup;
this.resFile = ctx.resFile;
this.resDir = ctx.resDir;
@@ -378,7 +378,7 @@ public class Context {
this.statsSource = ctx.statsSource;
this.executionIndex = ctx.executionIndex;
this.viewsTokenRewriteStreams = new HashMap<>();
- this.rewrittenStatementContexts = new HashSet<>();
+ this.subContexts = new HashSet<>();
this.opContext = new CompilationOpContext();
}
@@ -857,7 +857,7 @@ public class Context {
public void clear(boolean deleteResultDir) throws IOException {
// First clear the other contexts created by this query
- for (Context subContext : rewrittenStatementContexts) {
+ for (Context subContext : subContexts) {
subContext.clear();
}
// Then clear this context
@@ -1057,8 +1057,8 @@ public class Context {
}
}
- public void addRewrittenStatementContext(Context context) {
- rewrittenStatementContexts.add(context);
+ public void addSubContext(Context context) {
+ subContexts.add(context);
}
public void addCS(String path, ContentSummary cs) {
@@ -1132,7 +1132,7 @@ public class Context {
return this.calcitePlan;
}
- for (Context context : rewrittenStatementContexts) {
+ for (Context context : subContexts) {
if (context.calcitePlan != null) {
return context.calcitePlan;
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index 517b0cc..e70c92e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -683,7 +683,7 @@ public class Driver implements IDriver {
// and then, we acquire locks. If snapshot is still valid, we continue as usual.
// But if snapshot is not valid, we recompile the query.
driverContext.setRetrial(true);
- driverContext.getBackupContext().addRewrittenStatementContext(context);
+ driverContext.getBackupContext().addSubContext(context);
driverContext.getBackupContext().setHiveLocks(context.getHiveLocks());
context = driverContext.getBackupContext();
driverContext.getConf().set(ValidTxnList.VALID_TXNS_KEY,
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/alter/rebuild/AlterMaterializedViewRebuildAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/alter/rebuild/AlterMaterializedViewRebuildAnalyzer.java
index d1d0e3c..7ff2d19 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/alter/rebuild/AlterMaterializedViewRebuildAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/alter/rebuild/AlterMaterializedViewRebuildAnalyzer.java
@@ -94,7 +94,7 @@ public class AlterMaterializedViewRebuildAnalyzer extends CalcitePlanner {
String rewrittenInsertStatement = String.format(REWRITTEN_INSERT_STATEMENT,
tableName.getEscapedNotEmptyDbTable(), viewText);
rewrittenAST = ParseUtils.parse(rewrittenInsertStatement, ctx);
- this.ctx.addRewrittenStatementContext(ctx);
+ this.ctx.addSubContext(ctx);
if (!this.ctx.isExplainPlan() && AcidUtils.isTransactionalTable(table)) {
// Acquire lock for the given materialized view. Only one rebuild per materialized view can be triggered at a
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java
index 997132f..d25cadf 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java
@@ -161,6 +161,7 @@ public class ColumnStatsAutoGatherContext {
throws IOException, ParseException, SemanticException {
// 1. initialization
Context ctx = new Context(conf);
+ origCtx.addSubContext(ctx);
ctx.setOpContext(origCtx.getOpContext());
ctx.setExplainConfig(origCtx.getExplainConfig());
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
index b8231d2..2787b47 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
@@ -36,12 +36,10 @@ import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.metadata.HiveUtils;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.plan.HiveOperation;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
-import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
@@ -420,7 +418,7 @@ public class ColumnStatsSemanticAnalyzer extends SemanticAnalyzer {
analyzeRewrite.setColName(colNames);
analyzeRewrite.setColType(colType);
qbp.setAnalyzeRewrite(analyzeRewrite);
- origCtx.addRewrittenStatementContext(ctx);
+ origCtx.addSubContext(ctx);
initCtx(ctx);
ctx.setExplainConfig(origCtx.getExplainConfig());
LOG.info("Invoking analyze on rewritten query");
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
index 58b2615..5dac729 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
@@ -534,7 +534,7 @@ public class LoadSemanticAnalyzer extends SemanticAnalyzer {
rewrittenCtx = new Context(conf);
// We keep track of all the contexts that are created by this query
// so we can clear them when we finish execution
- ctx.addRewrittenStatementContext(rewrittenCtx);
+ ctx.addSubContext(rewrittenCtx);
} catch (IOException e) {
throw new SemanticException(ErrorMsg.LOAD_DATA_LAUNCH_JOB_IO_ERROR.getMsg());
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/RewriteSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/RewriteSemanticAnalyzer.java
index 7b25030..0b19f17 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/RewriteSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/RewriteSemanticAnalyzer.java
@@ -262,7 +262,7 @@ public abstract class RewriteSemanticAnalyzer extends CalcitePlanner {
rewrittenCtx.setHDFSCleanup(true);
// We keep track of all the contexts that are created by this query
// so we can clear them when we finish execution
- ctx.addRewrittenStatementContext(rewrittenCtx);
+ ctx.addSubContext(rewrittenCtx);
} catch (IOException e) {
throw new SemanticException(ErrorMsg.UPDATEDELETE_IO_ERROR.getMsg());
}