You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by bi...@apache.org on 2015/09/09 03:50:56 UTC
tez git commit: TEZ-2788. Allow TezAnalyzerBase to parse
SimpleHistory logs (bikas)
Repository: tez
Updated Branches:
refs/heads/master c07d6b751 -> 56f7847d2
TEZ-2788. Allow TezAnalyzerBase to parse SimpleHistory logs (bikas)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/56f7847d
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/56f7847d
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/56f7847d
Branch: refs/heads/master
Commit: 56f7847d26418879cfa32d7033147c8f95c56469
Parents: c07d6b7
Author: Bikas Saha <bi...@apache.org>
Authored: Tue Sep 8 18:50:47 2015 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Tue Sep 8 18:50:47 2015 -0700
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../tez/analyzer/plugins/TezAnalyzerBase.java | 63 +++++++++++++++++---
2 files changed, 55 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/56f7847d/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 4717249..77535b1 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ Release 0.8.1: Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-2788. Allow TezAnalyzerBase to parse SimpleHistory logs
TEZ-2782. VertexInfo.getAvgExecutionTimeInterval throws NPE when task does not have any valid attempts info
TEZ-2778. Improvements to handle multiple read errors with complex DAGs
TEZ-2768. Log a useful error message when the summary stream cannot be closed when shutting
http://git-wip-us.apache.org/repos/asf/tez/blob/56f7847d/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TezAnalyzerBase.java
----------------------------------------------------------------------
diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TezAnalyzerBase.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TezAnalyzerBase.java
index 3eb2f57..fa3bd97 100644
--- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TezAnalyzerBase.java
+++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TezAnalyzerBase.java
@@ -27,9 +27,12 @@ import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.Tool;
import org.apache.tez.analyzer.Analyzer;
+import org.apache.tez.history.ATSImportTool;
import org.apache.tez.history.parser.ATSFileParser;
+import org.apache.tez.history.parser.SimpleHistoryParser;
import org.apache.tez.history.parser.datamodel.DagInfo;
import com.google.common.base.Preconditions;
@@ -37,9 +40,10 @@ import com.google.common.base.Preconditions;
public abstract class TezAnalyzerBase extends Configured implements Tool, Analyzer {
- private static final String ATS_FILE_NAME = "atsFileName";
+ private static final String EVENT_FILE_NAME = "eventFileName";
private static final String OUTPUT_DIR = "outputDir";
private static final String DAG_ID = "dagId";
+ private static final String FROM_SIMPLE_HISTORY = "fromSimpleHistory";
private static final String HELP = "help";
private String outputDir;
@@ -52,10 +56,16 @@ public abstract class TezAnalyzerBase extends Configured implements Tool, Analyz
Option outputDirOption = OptionBuilder.withArgName(OUTPUT_DIR).withLongOpt(OUTPUT_DIR)
.withDescription("Directory to write outputs to.").hasArg().isRequired(false).create();
- Option inputATSFileNameOption = OptionBuilder.withArgName(ATS_FILE_NAME).withLongOpt
- (ATS_FILE_NAME)
- .withDescription("File with ATS data for the DAG").hasArg()
- .isRequired(true).create();
+ Option eventFileNameOption = OptionBuilder.withArgName(EVENT_FILE_NAME).withLongOpt
+ (EVENT_FILE_NAME)
+ .withDescription("File with event data for the DAG").hasArg()
+ .isRequired(false).create();
+
+ Option fromSimpleHistoryOption = OptionBuilder.withArgName(FROM_SIMPLE_HISTORY).withLongOpt
+ (FROM_SIMPLE_HISTORY)
+ .withDescription("Event data from Simple History logging. Must also specify event file")
+ .isRequired(false).create();
+
Option help = OptionBuilder.withArgName(HELP).withLongOpt
(HELP)
.withDescription("print help")
@@ -64,7 +74,8 @@ public abstract class TezAnalyzerBase extends Configured implements Tool, Analyz
Options opts = new Options();
opts.addOption(dagIdOption);
opts.addOption(outputDirOption);
- opts.addOption(inputATSFileNameOption);
+ opts.addOption(eventFileNameOption);
+ opts.addOption(fromSimpleHistoryOption);
opts.addOption(help);
return opts;
}
@@ -103,12 +114,46 @@ public abstract class TezAnalyzerBase extends Configured implements Tool, Analyz
if (outputDir == null) {
outputDir = System.getProperty("user.dir");
}
+
+ File file = null;
+ if (cmdLine.hasOption(EVENT_FILE_NAME)) {
+ file = new File(cmdLine.getOptionValue(EVENT_FILE_NAME));
+ }
- File file = new File(cmdLine.getOptionValue(ATS_FILE_NAME));
String dagId = cmdLine.getOptionValue(DAG_ID);
- ATSFileParser parser = new ATSFileParser(file);
- DagInfo dagInfo = parser.getDAGData(dagId);
+ DagInfo dagInfo = null;
+
+ if (file == null) {
+ if (cmdLine.hasOption(FROM_SIMPLE_HISTORY)) {
+ System.err.println("Event file name must be specified when using simple history");
+ printUsage();
+ return -2;
+ }
+ // using ATS - try to download directly
+ String[] importArgs = { "--dagId=" + dagId, "--downloadDir=" + outputDir };
+
+ int result = ATSImportTool.process(importArgs);
+ if (result != 0) {
+ System.err.println("Error downloading data from ATS");
+ return -3;
+ }
+
+ //Parse ATS data and verify results
+ //Parse downloaded contents
+ file = new File(outputDir
+ + Path.SEPARATOR + dagId
+ + Path.SEPARATOR + dagId + ".zip");
+ }
+
+ Preconditions.checkState(file != null);
+ if (!cmdLine.hasOption(FROM_SIMPLE_HISTORY)) {
+ ATSFileParser parser = new ATSFileParser(file);
+ dagInfo = parser.getDAGData(dagId);
+ } else {
+ SimpleHistoryParser parser = new SimpleHistoryParser(file);
+ dagInfo = parser.getDAGData(dagId);
+ }
Preconditions.checkState(dagInfo.getDagId().equals(dagId));
analyze(dagInfo);
return 0;