You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sa...@apache.org on 2018/08/13 05:54:44 UTC
hive git commit: HIVE-20329: Long running repl load (incr/bootstrap)
causing OOM error (Mahesh Kumar Behera, reviewed by Sankar Hariappan)
Repository: hive
Updated Branches:
refs/heads/master 250e10ecf -> 681051f10
HIVE-20329: Long running repl load (incr/bootstrap) causing OOM error (Mahesh Kumar Behera, reviewed by Sankar Hariappan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/681051f1
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/681051f1
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/681051f1
Branch: refs/heads/master
Commit: 681051f1080ad5eed4ac8a8e5daf187ff23d4839
Parents: 250e10e
Author: Sankar Hariappan <sa...@apache.org>
Authored: Mon Aug 13 11:24:28 2018 +0530
Committer: Sankar Hariappan <sa...@apache.org>
Committed: Mon Aug 13 11:24:28 2018 +0530
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java | 6 ++++++
.../apache/hadoop/hive/ql/exec/repl/ReplLoadWork.java | 11 +++++++++++
2 files changed, 17 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/681051f1/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java
index 6b5fc54..aa41301 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java
@@ -72,6 +72,12 @@ public class ReplLoadTask extends Task<ReplLoadWork> implements Serializable {
@Override
protected int execute(DriverContext driverContext) {
+ Task<? extends Serializable> rootTask = work.getRootTask();
+ if (rootTask != null) {
+ rootTask.setChildTasks(null);
+ }
+ work.setRootTask(this);
+ this.parentTasks = null;
if (work.isIncrementalLoad()) {
return executeIncrementalLoad(driverContext);
} else {
http://git-wip-us.apache.org/repos/asf/hive/blob/681051f1/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadWork.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadWork.java
index 8921e94..fdbcb15 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadWork.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hive.ql.exec.repl.incremental.IncrementalLoadEventsIter
import org.apache.hadoop.hive.ql.exec.repl.incremental.IncrementalLoadTasksBuilder;
import org.apache.hadoop.hive.ql.plan.Explain;
import org.apache.hadoop.hive.ql.session.LineageState;
+import org.apache.hadoop.hive.ql.exec.Task;
import java.io.IOException;
import java.io.Serializable;
@@ -42,6 +43,7 @@ public class ReplLoadWork implements Serializable {
private int loadTaskRunCount = 0;
private DatabaseEvent.State state = null;
private final transient IncrementalLoadTasksBuilder incrementalLoad;
+ private transient Task<? extends Serializable> rootTask;
/*
these are sessionState objects that are copied over to work to allow for parallel execution.
@@ -56,6 +58,7 @@ public class ReplLoadWork implements Serializable {
sessionStateLineageState = lineageState;
this.dumpDirectory = dumpDirectory;
this.dbNameToLoadIn = dbNameToLoadIn;
+ rootTask = null;
if (isIncrementalDump) {
incrementalIterator = new IncrementalLoadEventsIterator(dumpDirectory, hiveConf);
this.bootstrapIterator = null;
@@ -110,4 +113,12 @@ public class ReplLoadWork implements Serializable {
public IncrementalLoadTasksBuilder getIncrementalLoadTaskBuilder() {
return incrementalLoad;
}
+
+ public Task<? extends Serializable> getRootTask() {
+ return rootTask;
+ }
+
+ public void setRootTask(Task<? extends Serializable> rootTask) {
+ this.rootTask = rootTask;
+ }
}