You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ab...@apache.org on 2021/02/08 20:29:59 UTC
[tez] branch master updated: TEZ-4281: dag_*_priority.dot files
should go into a valid log directory (#102) (Laszlo Bodor reviewed by
Rajesh Balamohan)
This is an automated email from the ASF dual-hosted git repository.
abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tez.git
The following commit(s) were added to refs/heads/master by this push:
new 1847817 TEZ-4281: dag_*_priority.dot files should go into a valid log directory (#102) (Laszlo Bodor reviewed by Rajesh Balamohan)
1847817 is described below
commit 1847817c2a92f26b5e9aa71c250f4559a564bf0a
Author: Bodor Laszlo <bo...@gmail.com>
AuthorDate: Mon Feb 8 21:29:49 2021 +0100
TEZ-4281: dag_*_priority.dot files should go into a valid log directory (#102) (Laszlo Bodor reviewed by Rajesh Balamohan)
---
tez-dag/findbugs-exclude.xml | 12 ++++++++++++
tez-dag/src/main/java/org/apache/tez/Utils.java | 18 +++---------------
.../main/java/org/apache/tez/dag/app/DAGAppMaster.java | 2 +-
.../java/org/apache/tez/dag/app/dag/impl/DAGImpl.java | 18 +++++++++++++++++-
4 files changed, 33 insertions(+), 17 deletions(-)
diff --git a/tez-dag/findbugs-exclude.xml b/tez-dag/findbugs-exclude.xml
index a6ce380..50422ff 100644
--- a/tez-dag/findbugs-exclude.xml
+++ b/tez-dag/findbugs-exclude.xml
@@ -252,4 +252,16 @@
<Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD"/>
</Match>
+ <!-- TEZ-4281 -->
+ <Match>
+ <Class name="org.apache.tez.dag.app.dag.impl.DAGImpl"/>
+ <Method name="getLogDirs"/>
+ <Bug pattern="EI_EXPOSE_REP"/>
+ </Match>
+ <Match>
+ <Class name="org.apache.tez.dag.app.dag.impl.DAGImpl"/>
+ <Method name="setLogDirs"/>
+ <Bug pattern="EI_EXPOSE_REP2"/>
+ </Match>
+
</FindBugsFilter>
diff --git a/tez-dag/src/main/java/org/apache/tez/Utils.java b/tez-dag/src/main/java/org/apache/tez/Utils.java
index dbde327..b352334 100644
--- a/tez-dag/src/main/java/org/apache/tez/Utils.java
+++ b/tez-dag/src/main/java/org/apache/tez/Utils.java
@@ -16,9 +16,7 @@ package org.apache.tez;
import javax.annotation.Nullable;
import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.event.Event;
-import org.apache.tez.common.TezCommonUtils;
import org.apache.tez.dag.api.records.DAGProtos;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.dag.DAG;
@@ -113,19 +111,6 @@ public class Utils {
* Generate a visualization file.
* @param dag DAG.
* @param dagPB DAG plan.
- * @param scheduler scheduler that provide the priorities of the vertexes.
- */
- public static void generateDAGVizFile(final DAG dag,
- final DAGProtos.DAGPlan dagPB, @Nullable final DAGScheduler scheduler) {
- generateDAGVizFile(dag, dagPB, TezCommonUtils.getTrimmedStrings(
- System.getenv(ApplicationConstants.Environment.LOG_DIRS.name())),
- scheduler);
- }
-
- /**
- * Generate a visualization file.
- * @param dag DAG.
- * @param dagPB DAG plan.
* @param logDirs directories where the file will be written.
* @param scheduler scheduler that will provide the priorities
* of the vertexes.
@@ -217,6 +202,9 @@ public class Utils {
if (logDirs != null && logDirs.length != 0) {
outputFile += logDirs[0];
outputFile += File.separator;
+ } else {
+ LOG.warn("DAGVizFile will be created under current (.) directory: {},"
+ + " which is not expected and recommended", new File(".").getAbsolutePath());
}
outputFile += dagId.toString();
// Means we have set the priorities
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 395e84a..1352b68 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -1030,7 +1030,7 @@ public class DAGAppMaster extends AbstractService {
new DAGImpl(dagId, amConf, dagPB, dispatcher.getEventHandler(),
taskCommunicatorManager, dagCredentials, clock,
appMasterUgi.getShortUserName(),
- taskHeartbeatHandler, context);
+ taskHeartbeatHandler, context).setLogDirs(logDirs);
try {
if (LOG.isDebugEnabled()) {
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index 09e9e71..026ca29 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -57,6 +57,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.state.InvalidStateTransitonException;
@@ -67,6 +68,7 @@ import org.apache.hadoop.yarn.state.StateMachineFactory;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.tez.common.ATSConstants;
import org.apache.tez.common.ReflectionUtils;
+import org.apache.tez.common.TezCommonUtils;
import org.apache.tez.common.counters.AggregateTezCounters;
import org.apache.tez.common.counters.DAGCounter;
import org.apache.tez.common.counters.TezCounters;
@@ -225,6 +227,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
private TaskSpecificLaunchCmdOption taskSpecificLaunchCmdOption;
private static final DagStateChangedCallback STATE_CHANGED_CALLBACK = new DagStateChangedCallback();
+ private String[] logDirs;
@VisibleForTesting
Map<OutputKey, ListenableFuture<Void>> commitFutures
@@ -1662,7 +1665,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
// which didn't have the priorities
if (getConf().getBoolean(TezConfiguration.TEZ_GENERATE_DEBUG_ARTIFACTS,
TezConfiguration.TEZ_GENERATE_DEBUG_ARTIFACTS_DEFAULT)) {
- Utils.generateDAGVizFile(this, jobPlan, dagScheduler);
+ Utils.generateDAGVizFile(this, jobPlan, logDirs, dagScheduler);
}
return DAGState.INITED;
}
@@ -2510,4 +2513,17 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
eventHandler.handle(new DAGEventCommitCompleted(dagId, outputKey, false, t));
}
}
+
+ public String[] getLogDirs() {
+ if (logDirs == null) {
+ logDirs = TezCommonUtils
+ .getTrimmedStrings(System.getenv(ApplicationConstants.Environment.LOG_DIRS.name()));
+ }
+ return logDirs;
+ }
+
+ public DAGImpl setLogDirs(String[] logDirs) {
+ this.logDirs = logDirs;
+ return this;
+ }
}