You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by th...@apache.org on 2017/08/16 18:01:08 UTC
hive git commit: HIVE-17195 : Long chain of tasks created by REPL
LOAD shouldn't cause stack corruption. (Sankar Hariappan,
reviewed by Anishek Agarwal, Thejas Nair)
Repository: hive
Updated Branches:
refs/heads/master e9347ab97 -> 55ec87b59
HIVE-17195 : Long chain of tasks created by REPL LOAD shouldn't cause stack corruption. (Sankar Hariappan, reviewed by Anishek Agarwal, Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/55ec87b5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/55ec87b5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/55ec87b5
Branch: refs/heads/master
Commit: 55ec87b59630d1905688214e8b9409a06e436304
Parents: e9347ab
Author: Sankar Hariappan <ma...@gmail.com>
Authored: Wed Aug 16 11:00:56 2017 -0700
Committer: Thejas M Nair <th...@hortonworks.com>
Committed: Wed Aug 16 11:00:56 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/exec/NodeUtils.java | 68 +++++++++++++-------
.../apache/hadoop/hive/ql/exec/Utilities.java | 48 +++++++++-----
2 files changed, 76 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/55ec87b5/ql/src/java/org/apache/hadoop/hive/ql/exec/NodeUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/NodeUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/NodeUtils.java
index 5aae311..249ffb3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/NodeUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/NodeUtils.java
@@ -20,8 +20,11 @@ package org.apache.hadoop.hive.ql.exec;
import org.apache.hadoop.hive.ql.lib.Node;
+import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashSet;
+import java.util.List;
import java.util.Set;
/**
@@ -29,49 +32,64 @@ import java.util.Set;
*/
public class NodeUtils {
+
public static <T> void iterateTask(Collection<Task<?>> tasks, Class<T> clazz, Function<T> function) {
+ // Does a breadth first traversal of the tasks
Set<Task> visited = new HashSet<Task>();
- for (Task<?> task : tasks) {
- iterateTask(task, clazz, function, visited);
+ while (!tasks.isEmpty()) {
+ tasks = iterateTask(tasks, clazz, function, visited);
}
return;
}
- private static <T> void iterateTask(Task<?> task, Class<T> clazz, Function<T> function, Set<Task> visited) {
- if (!visited.add(task)) {
- return;
- }
- if (clazz.isInstance(task)) {
- function.apply(clazz.cast(task));
- }
- // this is for ConditionalTask
- if (task.getDependentTasks() != null) {
- for (Task<?> dependent : task.getDependentTasks()) {
- iterateTask(dependent, clazz, function, visited);
+ private static <T> Collection<Task<?>> iterateTask(Collection<Task<?>> tasks,
+ Class<T> clazz,
+ Function<T> function,
+ Set<Task> visited) {
+ Collection<Task<?>> childTasks = new ArrayList<>();
+ for (Task<?> task : tasks) {
+ if (!visited.add(task)) {
+ continue;
+ }
+ if (clazz.isInstance(task)) {
+ function.apply(clazz.cast(task));
+ }
+ // this is for ConditionalTask
+ if (task.getDependentTasks() != null) {
+ childTasks.addAll(task.getDependentTasks());
}
}
+ return childTasks;
}
public static <T> void iterate(Collection<? extends Node> nodes, Class<T> clazz, Function<T> function) {
Set<Node> visited = new HashSet<Node>();
- for (Node task : nodes) {
- iterate(task, clazz, function, visited);
+ List<Collection<? extends Node>> listNodes = Collections.singletonList(nodes);
+ while (!listNodes.isEmpty()) {
+ listNodes = iterate(listNodes, clazz, function, visited);
}
return;
}
- private static <T> void iterate(Node node, Class<T> clazz, Function<T> function, Set<Node> visited) {
- if (!visited.add(node)) {
- return;
- }
- if (clazz.isInstance(node)) {
- function.apply(clazz.cast(node));
- }
- if (node.getChildren() != null) {
- for (Node child : node.getChildren()) {
- iterate(child, clazz, function, visited);
+ private static <T> List<Collection<? extends Node>> iterate(List<Collection<? extends Node>> listNodes,
+ Class<T> clazz,
+ Function<T> function,
+ Set<Node> visited) {
+ List<Collection<? extends Node>> childListNodes = new ArrayList<>();
+ for (Collection<? extends Node> nodes : listNodes) {
+ for (Node node : nodes) {
+ if (!visited.add(node)) {
+ continue;
+ }
+ if (clazz.isInstance(node)) {
+ function.apply(clazz.cast(node));
+ }
+ if (node.getChildren() != null) {
+ childListNodes.add(node.getChildren());
+ }
}
}
+ return childListNodes;
}
public static interface Function<T> {
http://git-wip-us.apache.org/repos/asf/hive/blob/55ec87b5/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index eaf210f..99c85f2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -2434,15 +2434,20 @@ public final class Utilities {
public static List<TezTask> getTezTasks(List<Task<? extends Serializable>> tasks) {
List<TezTask> tezTasks = new ArrayList<TezTask>();
- Set<Task<? extends Serializable>> visited = new HashSet<Task<? extends Serializable>>();
if (tasks != null) {
- getTezTasks(tasks, tezTasks, visited);
+ Set<Task<? extends Serializable>> visited = new HashSet<Task<? extends Serializable>>();
+ while (!tasks.isEmpty()) {
+ tasks = getTezTasks(tasks, tezTasks, visited);
+ }
}
return tezTasks;
}
- private static void getTezTasks(List<Task<? extends Serializable>> tasks, List<TezTask> tezTasks,
- Set<Task<? extends Serializable>> visited) {
+ private static List<Task<? extends Serializable>> getTezTasks(
+ List<Task<? extends Serializable>> tasks,
+ List<TezTask> tezTasks,
+ Set<Task<? extends Serializable>> visited) {
+ List<Task<? extends Serializable>> childTasks = new ArrayList<>();
for (Task<? extends Serializable> task : tasks) {
if (visited.contains(task)) {
continue;
@@ -2452,23 +2457,29 @@ public final class Utilities {
}
if (task.getDependentTasks() != null) {
- getTezTasks(task.getDependentTasks(), tezTasks, visited);
+ childTasks.addAll(task.getDependentTasks());
}
visited.add(task);
}
+ return childTasks;
}
public static List<SparkTask> getSparkTasks(List<Task<? extends Serializable>> tasks) {
List<SparkTask> sparkTasks = new ArrayList<SparkTask>();
- Set<Task<? extends Serializable>> visited = new HashSet<Task<? extends Serializable>>();
if (tasks != null) {
- getSparkTasks(tasks, sparkTasks, visited);
+ Set<Task<? extends Serializable>> visited = new HashSet<Task<? extends Serializable>>();
+ while (!tasks.isEmpty()) {
+ tasks = getSparkTasks(tasks, sparkTasks, visited);
+ }
}
return sparkTasks;
}
- private static void getSparkTasks(List<Task<? extends Serializable>> tasks,
- List<SparkTask> sparkTasks, Set<Task<? extends Serializable>> visited) {
+ private static List<Task<? extends Serializable>> getSparkTasks(
+ List<Task<? extends Serializable>> tasks,
+ List<SparkTask> sparkTasks,
+ Set<Task<? extends Serializable>> visited) {
+ List<Task<? extends Serializable>> childTasks = new ArrayList<>();
for (Task<? extends Serializable> task : tasks) {
if (visited.contains(task)) {
continue;
@@ -2478,23 +2489,29 @@ public final class Utilities {
}
if (task.getDependentTasks() != null) {
- getSparkTasks(task.getDependentTasks(), sparkTasks, visited);
+ childTasks.addAll(task.getDependentTasks());
}
visited.add(task);
}
+ return childTasks;
}
public static List<ExecDriver> getMRTasks(List<Task<? extends Serializable>> tasks) {
List<ExecDriver> mrTasks = new ArrayList<ExecDriver>();
- Set<Task<? extends Serializable>> visited = new HashSet<Task<? extends Serializable>>();
if (tasks != null) {
- getMRTasks(tasks, mrTasks, visited);
+ Set<Task<? extends Serializable>> visited = new HashSet<Task<? extends Serializable>>();
+ while (!tasks.isEmpty()) {
+ tasks = getMRTasks(tasks, mrTasks, visited);
+ }
}
return mrTasks;
}
- private static void getMRTasks(List<Task<? extends Serializable>> tasks, List<ExecDriver> mrTasks,
- Set<Task<? extends Serializable>> visited) {
+ private static List<Task<? extends Serializable>> getMRTasks(
+ List<Task<? extends Serializable>> tasks,
+ List<ExecDriver> mrTasks,
+ Set<Task<? extends Serializable>> visited) {
+ List<Task<? extends Serializable>> childTasks = new ArrayList<>();
for (Task<? extends Serializable> task : tasks) {
if (visited.contains(task)) {
continue;
@@ -2504,10 +2521,11 @@ public final class Utilities {
}
if (task.getDependentTasks() != null) {
- getMRTasks(task.getDependentTasks(), mrTasks, visited);
+ childTasks.addAll(task.getDependentTasks());
}
visited.add(task);
}
+ return childTasks;
}
/**