You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2020/02/21 16:39:03 UTC
[hive] branch master updated: HIVE-22744 : TezTask for the vertex
with more than one outedge should have proportional sort memory (Ramesh
Kumar via Ashutosh Chauhan)
This is an automated email from the ASF dual-hosted git repository.
hashutosh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new c3f3523 HIVE-22744 : TezTask for the vertex with more than one outedge should have proportional sort memory (Ramesh Kumar via Ashutosh Chauhan)
c3f3523 is described below
commit c3f3523b498748f4a309df807893e0a4f94c1338
Author: Ramesh Kumar Thangarajan <ra...@cloudera.com>
AuthorDate: Fri Feb 21 08:38:05 2020 -0800
HIVE-22744 : TezTask for the vertex with more than one outedge should have proportional sort memory (Ramesh Kumar via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
---
ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
index 4002ccb..0bacb05 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
@@ -82,6 +82,7 @@ import org.apache.tez.dag.api.client.DAGClient;
import org.apache.tez.dag.api.client.DAGStatus;
import org.apache.tez.dag.api.client.StatusGetOpts;
import org.apache.tez.dag.api.client.VertexStatus;
+import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -479,6 +480,18 @@ public class TezTask extends Task<TezWork> {
checkOutputSpec(w, wxConf);
Vertex wx = utils.createVertex(wxConf, w, scratchDir, fs, ctx, !isFinal,
work, work.getVertexType(w), vertexResources);
+ if (work.getChildren(w).size() > 1) {
+ String value = wxConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB);
+ int originalValue = 0;
+ if(value == null) {
+ originalValue = TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB_DEFAULT;
+ } else {
+ originalValue = Integer.valueOf(value);
+ }
+ int newValue = (int) (originalValue / work.getChildren(w).size());
+ wxConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, Integer.toString(newValue));
+ LOG.info("Modified " + TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB + " to " + newValue);
+ }
if (w.getReservedMemoryMB() > 0) {
// If reversedMemoryMB is set, make memory allocation fraction adjustment as needed
double frac = DagUtils.adjustMemoryReserveFraction(w.getReservedMemoryMB(), super.conf);