You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by je...@apache.org on 2014/11/17 20:22:10 UTC
[01/50] [abbrv] tez git commit: TEZ-792. Default staging path should
have user name. (Mohammad Kamrul Islam via hitesh)
Repository: tez
Updated Branches:
refs/heads/TEZ-8 2ce8ebe60 -> 052f71cc7
TEZ-792. Default staging path should have user name. (Mohammad Kamrul Islam via hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/9c166076
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/9c166076
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/9c166076
Branch: refs/heads/TEZ-8
Commit: 9c166076ec7e8ff16deeba22c6c1b2015c2d73e2
Parents: 3a5f6e3
Author: Hitesh Shah <hi...@apache.org>
Authored: Tue Oct 28 21:56:33 2014 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Tue Oct 28 21:56:33 2014 -0700
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../src/main/java/org/apache/tez/dag/api/TezConfiguration.java | 3 ++-
2 files changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/9c166076/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 5dafcb9..6bf95ae 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -73,6 +73,7 @@ ALL CHANGES:
TEZ-1664. Add checks to ensure that the client and AM are compatible.
TEZ-1689. Exception handling for EdgeManagerPlugin.
TEZ-1701. ATS fixes to flush all history events and also using batching.
+ TEZ-792. Default staging path should have user name.
Release 0.5.1: 2014-10-02
http://git-wip-us.apache.org/repos/asf/tez/blob/9c166076/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index e3c3c83..1919f5c 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -100,7 +100,8 @@ public class TezConfiguration extends Configuration {
* String value. Specifies a directory where Tez can create temporary job artifacts.
*/
public static final String TEZ_AM_STAGING_DIR = TEZ_PREFIX + "staging-dir";
- public static final String TEZ_AM_STAGING_DIR_DEFAULT = "/tmp/tez/staging";
+ public static final String TEZ_AM_STAGING_DIR_DEFAULT = "/tmp/"
+ + System.getProperty("user.name") + "/tez/staging";
/**
* String value that is a file path.
[23/50] [abbrv] tez git commit: TEZ-1743. Add versions-maven-plugins
artifacts to gitignore (jeagles)
Posted by je...@apache.org.
TEZ-1743. Add versions-maven-plugins artifacts to gitignore (jeagles)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/f4d50f81
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/f4d50f81
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/f4d50f81
Branch: refs/heads/TEZ-8
Commit: f4d50f817ee913cac6055b1f244b948f9fcb4903
Parents: 93687ae
Author: Jonathan Eagles <je...@gmail.com>
Authored: Wed Nov 5 21:46:45 2014 -0600
Committer: Jonathan Eagles <je...@gmail.com>
Committed: Wed Nov 5 21:46:45 2014 -0600
----------------------------------------------------------------------
.gitignore | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/f4d50f81/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index e5a919d..c56cd1c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,4 +6,5 @@
.classpath
.project
.settings
+pom.xml.versionsBackup
target
[14/50] [abbrv] tez git commit: TEZ-1725. Fix nanosecond to millis
conversion in TezMxBeanResourceCalculator (Rajesh Balamohan)
Posted by je...@apache.org.
TEZ-1725. Fix nanosecond to millis conversion in TezMxBeanResourceCalculator (Rajesh Balamohan)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/b66ff72c
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/b66ff72c
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/b66ff72c
Branch: refs/heads/TEZ-8
Commit: b66ff72cc02d333d36c312562b7f06ac174e9830
Parents: 81322ef
Author: Rajesh Balamohan <rb...@apache.org>
Authored: Fri Oct 31 12:22:59 2014 +0530
Committer: Rajesh Balamohan <rb...@apache.org>
Committed: Fri Oct 31 12:22:59 2014 +0530
----------------------------------------------------------------------
CHANGES.txt | 1 +
tez-plugins/pom.xml | 4 +++-
.../java/org/apache/tez/util/TezMxBeanResourceCalculator.java | 5 +----
3 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/b66ff72c/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 3fdd96e..9c28485 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -95,6 +95,7 @@ ALL CHANGES:
TEZ-1703. Exception handling for InputInitializer.
TEZ-1698. Cut down on ResourceCalculatorProcessTree overheads in Tez.
TEZ-1703. addendum - fix flaky test.
+ TEZ-1725. Fix nanosecond to millis conversion in TezMxBeanResourceCalculator.
Release 0.5.1: 2014-10-02
http://git-wip-us.apache.org/repos/asf/tez/blob/b66ff72c/tez-plugins/pom.xml
----------------------------------------------------------------------
diff --git a/tez-plugins/pom.xml b/tez-plugins/pom.xml
index 7c15215..b6ce081 100644
--- a/tez-plugins/pom.xml
+++ b/tez-plugins/pom.xml
@@ -30,7 +30,9 @@
<profile>
<id>hadoop24</id>
<activation>
- <activeByDefault>true</activeByDefault>
+ <property>
+ <name>!skipATS</name>
+ </property>
</activation>
<modules>
<module>tez-yarn-timeline-history</module>
http://git-wip-us.apache.org/repos/asf/tez/blob/b66ff72c/tez-plugins/tez-mbeans-resource-calculator/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-mbeans-resource-calculator/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java b/tez-plugins/tez-mbeans-resource-calculator/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java
index 330a2fd..4aa8f85 100644
--- a/tez-plugins/tez-mbeans-resource-calculator/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java
+++ b/tez-plugins/tez-mbeans-resource-calculator/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java
@@ -31,7 +31,6 @@ public class TezMxBeanResourceCalculator extends ResourceCalculatorProcessTree {
private final com.sun.management.OperatingSystemMXBean osBean;
private final Runtime runtime;
- private final AtomicLong cumulativeCPU;
/**
* Create process-tree instance with specified root process.
@@ -45,7 +44,6 @@ public class TezMxBeanResourceCalculator extends ResourceCalculatorProcessTree {
runtime = Runtime.getRuntime();
osBean =
(com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
- cumulativeCPU = new AtomicLong();
}
@Override public void updateProcessTree() {
@@ -67,8 +65,7 @@ public class TezMxBeanResourceCalculator extends ResourceCalculatorProcessTree {
@Override public long getCumulativeCpuTime() {
//convert to milliseconds
- return TimeUnit.MILLISECONDS.convert(cumulativeCPU.addAndGet(osBean.getProcessCpuTime()),
- TimeUnit.MILLISECONDS);
+ return TimeUnit.MILLISECONDS.convert(osBean.getProcessCpuTime(), TimeUnit.NANOSECONDS);
}
@Override public boolean checkPidPgrpidForMatch() {
[46/50] [abbrv] tez git commit: Update CHANGES.txt for TEZ-1774.
(hitesh)
Posted by je...@apache.org.
Update CHANGES.txt for TEZ-1774. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/f4c2112f
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/f4c2112f
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/f4c2112f
Branch: refs/heads/TEZ-8
Commit: f4c2112f647ce1aa7b7ab9febad2118bd9451cc6
Parents: f152a7f
Author: Hitesh Shah <hi...@apache.org>
Authored: Thu Nov 13 11:05:20 2014 -0800
Committer: Hitesh Shah <hi...@apache.org>
Committed: Thu Nov 13 11:05:20 2014 -0800
----------------------------------------------------------------------
CHANGES.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/f4c2112f/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index e73ab9b..4f0417e 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -10,7 +10,6 @@ ALL CHANGES:
TEZ-1738. Tez tfile parser for log parsing
TEZ-1627. Remove OUTPUT_CONSUMABLE and related Event in TaskAttemptImpl
TEZ-1736. Add support for Inputs/Outputs in runtime-library to generate history text data.
- TEZ-1774. AppLaunched event for Timeline does not have start time set.
Release 0.5.3: Unreleased
@@ -26,6 +25,7 @@ ALL CHANGES:
TEZ-1687. Use logIdentifier of Vertex for logging.
TEZ-1737. Should add taskNum in VertexFinishedEvent.
TEZ-1770. Handle ConnectExceptions correctly when establishing connections to an NM which may be down.
+ TEZ-1774. AppLaunched event for Timeline does not have start time set.
Release 0.5.2: 2014-11-07
[33/50] [abbrv] tez git commit: Fixing CHANGES.txt for TEZ-1749
Posted by je...@apache.org.
Fixing CHANGES.txt for TEZ-1749
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/0127a90a
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/0127a90a
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/0127a90a
Branch: refs/heads/TEZ-8
Commit: 0127a90aa84bce88d97ff5c632d38faf4aaaf7aa
Parents: 422d5d9
Author: Rajesh Balamohan <rb...@apache.org>
Authored: Sat Nov 8 11:24:01 2014 -0800
Committer: Rajesh Balamohan <rb...@apache.org>
Committed: Sat Nov 8 11:24:01 2014 -0800
----------------------------------------------------------------------
CHANGES.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/0127a90a/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index f849fec..da457df 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,7 +9,6 @@ ALL CHANGES:
TEZ-1733. TezMerger should sort FileChunks on size when merging
TEZ-1738. Tez tfile parser for log parsing
TEZ-1627. Remove OUTPUT_CONSUMABLE and related Event in TaskAttemptImpl
- TEZ-1749. Increase test timeout for TestLocalMode.testMultipleClientsWithSession
Release 0.5.3: Unreleased
@@ -18,6 +17,7 @@ ALL CHANGES:
TEZ-1745. TestATSHistoryLoggingService::testATSHistoryLoggingServiceShutdown can be flaky.
TEZ-1747. Increase test timeout for TestSecureShuffle.
TEZ-1746. Flaky test in TestVertexImpl and TestExceptionPropagation.
+ TEZ-1749. Increase test timeout for TestLocalMode.testMultipleClientsWithSession
Release 0.5.2: 2014-11-07
[22/50] [abbrv] tez git commit: TEZ-1733. TezMerger should sort
FileChunks on size when merging (Prakash Ramachandran via Rajesh Balamohan)
Posted by je...@apache.org.
TEZ-1733. TezMerger should sort FileChunks on size when merging (Prakash Ramachandran via Rajesh Balamohan)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/93687aee
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/93687aee
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/93687aee
Branch: refs/heads/TEZ-8
Commit: 93687aeec59263d949e8a66d6d78f812b93f3bdf
Parents: cd0ed75
Author: Rajesh Balamohan <rb...@apache.org>
Authored: Wed Nov 5 17:32:41 2014 -0800
Committer: Rajesh Balamohan <rb...@apache.org>
Committed: Wed Nov 5 17:34:12 2014 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../src/main/java/org/apache/hadoop/io/FileChunk.java | 12 ++++++------
2 files changed, 7 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/93687aee/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index adb4352..1b0a2d9 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -6,6 +6,7 @@ Release 0.6.0: Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-1733. TezMerger should sort FileChunks on size when merging
Release 0.5.2: Unreleased
http://git-wip-us.apache.org/repos/asf/tez/blob/93687aee/tez-runtime-library/src/main/java/org/apache/hadoop/io/FileChunk.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/hadoop/io/FileChunk.java b/tez-runtime-library/src/main/java/org/apache/hadoop/io/FileChunk.java
index 0ba39e2..e7a5c24 100644
--- a/tez-runtime-library/src/main/java/org/apache/hadoop/io/FileChunk.java
+++ b/tez-runtime-library/src/main/java/org/apache/hadoop/io/FileChunk.java
@@ -72,22 +72,22 @@ public class FileChunk implements Comparable<FileChunk> {
@Override
public int compareTo(FileChunk that) {
+ long lc;
+ lc = length - that.length;
+ if (lc != 0) {
+ return lc < 0 ? -1 : 1;
+ }
+
int c = path.compareTo(that.path);
if (c != 0) {
return c;
}
- long lc;
lc = offset - that.offset;
if (lc != 0) {
return lc < 0 ? -1 : 1;
}
- lc = length - that.length;
- if (lc != 0) {
- return lc < 0 ? -1 : 1;
- }
-
return 0;
}
[10/50] [abbrv] tez git commit: TEZ-1703. Exception handling for
InputInitializer. (zjffdu)
Posted by je...@apache.org.
TEZ-1703. Exception handling for InputInitializer. (zjffdu)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/4ec29425
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/4ec29425
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/4ec29425
Branch: refs/heads/TEZ-8
Commit: 4ec29425d63decfd8de4e1528f043271cf7cb3b2
Parents: 7f8fc75
Author: Jeff Zhang <zj...@apache.org>
Authored: Fri Oct 31 09:55:22 2014 +0800
Committer: Jeff Zhang <zj...@apache.org>
Committed: Fri Oct 31 09:55:22 2014 +0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../tez/runtime/api/InputInitializer.java | 3 +-
.../app/dag/RootInputInitializerManager.java | 30 +++-
.../dag/event/VertexEventRootInputFailed.java | 7 +-
.../apache/tez/dag/app/dag/impl/DAGImpl.java | 4 +-
.../apache/tez/dag/app/dag/impl/VertexImpl.java | 52 ++++--
.../tez/dag/app/dag/impl/TestVertexImpl.java | 178 ++++++++++++++++++-
.../tez/test/TestExceptionPropagation.java | 138 +++++++++++---
8 files changed, 362 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/4ec29425/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 0b2dc65..61b380e 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -92,6 +92,7 @@ ALL CHANGES:
TEZ-1716. Additional ATS data for UI.
TEZ-1722. DAG should be related to Application Id in ATS data.
TEZ-1711. Don't cache outputSpecList in VertexImpl.getOutputSpecList(taskIndex)
+ TEZ-1703. Exception handling for InputInitializer.
Release 0.5.1: 2014-10-02
http://git-wip-us.apache.org/repos/asf/tez/blob/4ec29425/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializer.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializer.java b/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializer.java
index 7b22b62..d9d6517 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializer.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializer.java
@@ -101,7 +101,8 @@ public abstract class InputInitializer {
* @param stateUpdate an event indicating the name of the vertex, and it's updated state.
* Additional information may be available for specific events, Look at the
* type hierarchy for {@link org.apache.tez.dag.api.event.VertexStateUpdate}
+ * @throws Exception
*/
- public void onVertexStateUpdated(VertexStateUpdate stateUpdate) {
+ public void onVertexStateUpdated(VertexStateUpdate stateUpdate) throws Exception {
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/4ec29425/tez-dag/src/main/java/org/apache/tez/dag/app/dag/RootInputInitializerManager.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/RootInputInitializerManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/RootInputInitializerManager.java
index 1f7a83f..bdd3689 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/RootInputInitializerManager.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/RootInputInitializerManager.java
@@ -19,6 +19,8 @@
package org.apache.tez.dag.app.dag;
import javax.annotation.Nullable;
+
+import java.lang.reflect.UndeclaredThrowableException;
import java.security.PrivilegedExceptionAction;
import java.util.Collection;
import java.util.HashMap;
@@ -53,7 +55,9 @@ import org.apache.tez.dag.api.oldrecords.TaskState;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.dag.event.VertexEventRootInputFailed;
import org.apache.tez.dag.app.dag.event.VertexEventRootInputInitialized;
+import org.apache.tez.dag.app.dag.impl.AMUserCodeException;
import org.apache.tez.dag.app.dag.impl.TezRootInputInitializerContextImpl;
+import org.apache.tez.dag.app.dag.impl.AMUserCodeException.Source;
import org.apache.tez.dag.records.TezTaskID;
import org.apache.tez.dag.records.TezVertexID;
import org.apache.tez.runtime.api.Event;
@@ -275,12 +279,18 @@ public class RootInputInitializerManager {
@SuppressWarnings("unchecked")
@Override
public void onFailure(Throwable t) {
+ // catch real root cause of failure, it would throw UndeclaredThrowableException
+ // if using UGI.doAs
+ if (t instanceof UndeclaredThrowableException) {
+ t = t.getCause();
+ }
initializer.setComplete();
LOG.info(
"Failed InputInitializer for Input: " + initializer.getInput().getName() +
" on vertex " + initializer.getVertexLogIdentifier());
eventHandler
- .handle(new VertexEventRootInputFailed(vertexID, initializer.getInput().getName(), t));
+ .handle(new VertexEventRootInputFailed(vertexID, initializer.getInput().getName(),
+ new AMUserCodeException(Source.InputInitializer,t)));
}
}
@@ -294,6 +304,7 @@ public class RootInputInitializerManager {
private final InputInitializerContext context;
private final AtomicBoolean isComplete = new AtomicBoolean(false);
private final String vertexLogIdentifier;
+ private final TezVertexID vertexId;
private final StateChangeNotifier stateChangeNotifier;
private final List<String> notificationRegisteredVertices = Lists.newArrayList();
private final AppContext appContext;
@@ -306,6 +317,7 @@ public class RootInputInitializerManager {
this.initializer = initializer;
this.context = context;
this.vertexLogIdentifier = vertex.getLogIdentifier();
+ this.vertexId = vertex.getVertexId();
this.stateChangeNotifier = stateChangeNotifier;
this.appContext = appContext;
}
@@ -348,6 +360,7 @@ public class RootInputInitializerManager {
}
}
+ @SuppressWarnings("unchecked")
@Override
public void onStateUpdated(VertexStateUpdate event) {
if (isComplete()) {
@@ -357,7 +370,13 @@ public class RootInputInitializerManager {
" since initializer " + input.getName() + " is already complete.");
}
} else {
- initializer.onVertexStateUpdated(event);
+ try {
+ initializer.onVertexStateUpdated(event);
+ } catch (Exception e) {
+ appContext.getEventHandler().handle(
+ new VertexEventRootInputFailed(vertexId, input.getName(),
+ new AMUserCodeException(Source.InputInitializer,e)));
+ }
}
}
@@ -455,14 +474,15 @@ public class RootInputInitializerManager {
sendEvents(toForwardEvents);
}
+ @SuppressWarnings("unchecked")
private void sendEvents(List<InputInitializerEvent> events) {
if (events != null && !events.isEmpty()) {
try {
initializer.handleInputInitializerEvent(events);
} catch (Exception e) {
- throw new TezUncheckedException(
- "Initializer for input: " + getInput().getName() + " on vertex: " + getVertexLogIdentifier() +
- " failed to process events", e);
+ appContext.getEventHandler().handle(
+ new VertexEventRootInputFailed(vertexId, input.getName(),
+ new AMUserCodeException(Source.InputInitializer,e)));
}
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/4ec29425/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventRootInputFailed.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventRootInputFailed.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventRootInputFailed.java
index 4ab4ae9..b7701d7 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventRootInputFailed.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventRootInputFailed.java
@@ -18,20 +18,21 @@
package org.apache.tez.dag.app.dag.event;
+import org.apache.tez.dag.app.dag.impl.AMUserCodeException;
import org.apache.tez.dag.records.TezVertexID;
public class VertexEventRootInputFailed extends VertexEvent {
private final String inputName;
- private final Throwable error;
+ private final AMUserCodeException error;
- public VertexEventRootInputFailed(TezVertexID vertexId, String inputName, Throwable error) {
+ public VertexEventRootInputFailed(TezVertexID vertexId, String inputName, AMUserCodeException error) {
super(vertexId, VertexEventType.V_ROOT_INPUT_FAILED);
this.inputName = inputName;
this.error = error;
}
- public Throwable getError() {
+ public AMUserCodeException getError() {
return this.error;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/4ec29425/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index 6dccf3a..cddcbd5 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -1686,9 +1686,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
}
else if (vertexEvent.getVertexState() == VertexState.FAILED) {
job.enactKill(
- DAGTerminationCause.VERTEX_FAILURE,
- vertexEvent.getVertexTerminationCause() == null ? VertexTerminationCause.OTHER_VERTEX_FAILURE
- : vertexEvent.getVertexTerminationCause());
+ DAGTerminationCause.VERTEX_FAILURE, VertexTerminationCause.OTHER_VERTEX_FAILURE);
job.vertexFailed(vertex);
forceTransitionToKillWait = true;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/4ec29425/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index c182810..4a88949 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -335,7 +335,8 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
EnumSet.of(VertexState.INITED, VertexState.FAILED),
VertexEventType.V_READY_TO_INIT,
new VertexInitializedTransition())
- .addTransition(VertexState.INITIALIZING, VertexState.FAILED,
+ .addTransition(VertexState.INITIALIZING,
+ EnumSet.of(VertexState.FAILED),
VertexEventType.V_ROOT_INPUT_FAILED,
new RootInputInitFailedTransition())
.addTransition(VertexState.INITIALIZING, VertexState.INITIALIZING,
@@ -367,6 +368,10 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
// Transitions from INITED state
// SOURCE_VERTEX_STARTED - for sources which determine parallelism,
// they must complete before this vertex can start.
+ .addTransition(VertexState.INITED,
+ EnumSet.of(VertexState.FAILED),
+ VertexEventType.V_ROOT_INPUT_FAILED,
+ new RootInputInitFailedTransition())
.addTransition
(VertexState.INITED,
EnumSet.of(VertexState.INITED, VertexState.ERROR),
@@ -399,6 +404,10 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
INTERNAL_ERROR_TRANSITION)
// Transitions from RUNNING state
+ .addTransition(VertexState.RUNNING,
+ EnumSet.of(VertexState.TERMINATING),
+ VertexEventType.V_ROOT_INPUT_FAILED,
+ new RootInputInitFailedTransition())
.addTransition(VertexState.RUNNING, VertexState.RUNNING,
VertexEventType.V_TASK_ATTEMPT_COMPLETED,
TASK_ATTEMPT_COMPLETED_EVENT_TRANSITION)
@@ -451,6 +460,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
// Ignore-able events
.addTransition(VertexState.TERMINATING, VertexState.TERMINATING,
EnumSet.of(VertexEventType.V_TERMINATE,
+ VertexEventType.V_ROOT_INPUT_FAILED,
VertexEventType.V_SOURCE_VERTEX_STARTED,
VertexEventType.V_ROOT_INPUT_INITIALIZED,
VertexEventType.V_NULL_EDGE_INITIALIZED,
@@ -483,6 +493,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
new TaskCompletedAfterVertexSuccessTransition())
.addTransition(VertexState.SUCCEEDED, VertexState.SUCCEEDED,
EnumSet.of(VertexEventType.V_TERMINATE,
+ VertexEventType.V_ROOT_INPUT_FAILED,
VertexEventType.V_TASK_ATTEMPT_COMPLETED,
// after we are done reruns of source tasks should not affect
// us. These reruns may be triggered by other consumer vertices.
@@ -501,6 +512,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
// Ignore-able events
.addTransition(VertexState.FAILED, VertexState.FAILED,
EnumSet.of(VertexEventType.V_TERMINATE,
+ VertexEventType.V_ROOT_INPUT_FAILED,
VertexEventType.V_SOURCE_VERTEX_STARTED,
VertexEventType.V_TASK_RESCHEDULED,
VertexEventType.V_START,
@@ -522,6 +534,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
// Ignore-able events
.addTransition(VertexState.KILLED, VertexState.KILLED,
EnumSet.of(VertexEventType.V_TERMINATE,
+ VertexEventType.V_ROOT_INPUT_FAILED,
VertexEventType.V_INIT,
VertexEventType.V_SOURCE_VERTEX_STARTED,
VertexEventType.V_START,
@@ -541,6 +554,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
VertexState.ERROR,
VertexState.ERROR,
EnumSet.of(VertexEventType.V_INIT,
+ VertexEventType.V_ROOT_INPUT_FAILED,
VertexEventType.V_SOURCE_VERTEX_STARTED,
VertexEventType.V_START,
VertexEventType.V_ROUTE_EVENT,
@@ -1654,7 +1668,18 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
}
else if (vertex.terminationCause == VertexTerminationCause.AM_USERCODE_FAILURE) {
vertex.setFinishTime();
- String diagnosticMsg = "Vertex failed/killed due to VertexManager failed. "
+ String diagnosticMsg = "Vertex failed/killed due to VertexManagerPlugin/EdgeManagerPlugin failed. "
+ + "failedTasks:"
+ + vertex.failedTaskCount
+ + " killedTasks:"
+ + vertex.killedTaskCount;
+ LOG.info(diagnosticMsg);
+ vertex.abortVertex(State.FAILED);
+ return vertex.finished(VertexState.FAILED);
+ }
+ else if (vertex.terminationCause == VertexTerminationCause.ROOT_INPUT_INIT_FAILURE) {
+ vertex.setFinishTime();
+ String diagnosticMsg = "Vertex failed/killed due to ROOT_INPUT_INIT_FAILURE failed. "
+ "failedTasks:"
+ vertex.failedTaskCount
+ " killedTasks:"
@@ -1665,7 +1690,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
}
else {
//should never occur
- throw new TezUncheckedException("All tasks complete, but cannot determine final state of vertex"
+ throw new TezUncheckedException("All tasks complete, but cannot determine final state of vertex:" + vertex.logIdentifier
+ ", failedTaskCount=" + vertex.failedTaskCount
+ ", killedTaskCount=" + vertex.killedTaskCount
+ ", successfulTaskCount=" + vertex.succeededTaskCount
@@ -3202,19 +3227,26 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
}
private static class RootInputInitFailedTransition implements
- SingleArcTransition<VertexImpl, VertexEvent> {
+ MultipleArcTransition<VertexImpl, VertexEvent, VertexState> {
@Override
- public void transition(VertexImpl vertex, VertexEvent event) {
+ public VertexState transition(VertexImpl vertex, VertexEvent event) {
VertexEventRootInputFailed fe = (VertexEventRootInputFailed) event;
String msg = "Vertex Input: " + fe.getInputName()
+ " initializer failed, vertex=" + vertex.getLogIdentifier();
- if (fe.getError() != null) {
- msg = msg + ExceptionUtils.getStackTrace(fe.getError());
+ LOG.error(msg, fe.getError());
+ if (vertex.getState() == VertexState.RUNNING) {
+ vertex.addDiagnostic(msg
+ + ", " + ExceptionUtils.getStackTrace(fe.getError().getCause()));
+ vertex.tryEnactKill(VertexTerminationCause.ROOT_INPUT_INIT_FAILURE,
+ TaskTerminationCause.AM_USERCODE_FAILURE);
+ return VertexState.TERMINATING;
+ } else {
+ vertex.finished(VertexState.FAILED,
+ VertexTerminationCause.ROOT_INPUT_INIT_FAILURE, msg
+ + ", " + ExceptionUtils.getStackTrace(fe.getError().getCause()));
+ return VertexState.FAILED;
}
- LOG.error(msg);
- vertex.finished(VertexState.FAILED,
- VertexTerminationCause.ROOT_INPUT_INIT_FAILURE, msg);
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/4ec29425/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
index fdf0e07..ef2c7bd 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
@@ -125,6 +125,7 @@ import org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted;
import org.apache.tez.dag.app.dag.event.VertexEventTaskReschedule;
import org.apache.tez.dag.app.dag.event.VertexEventTermination;
import org.apache.tez.dag.app.dag.event.VertexEventType;
+import org.apache.tez.dag.app.dag.impl.AMUserCodeException.Source;
import org.apache.tez.dag.app.dag.impl.DAGImpl.VertexGroupInfo;
import org.apache.tez.dag.app.dag.impl.TestVertexImpl.VertexManagerWithException.VMExceptionLocation;
import org.apache.tez.dag.app.rm.TaskSchedulerEventHandler;
@@ -4647,7 +4648,8 @@ public class TestVertexImpl {
super.runInputInitializers(inputs);
eventHandler.handle(new VertexEventRootInputFailed(vertexID, inputs
.get(0).getName(),
- new RuntimeException("MockInitializerFailed")));
+ new AMUserCodeException(Source.InputInitializer,
+ new RuntimeException("MockInitializerFailed"))));
dispatcher.await();
}
@@ -4969,9 +4971,164 @@ public class TestVertexImpl {
initVertex(v1);
String diagnostics = StringUtils.join(v1.getDiagnostics(), ",");
assertTrue(diagnostics.contains(IIExceptionLocation.Initialize.name()));
+ Assert.assertEquals(VertexState.FAILED, v1.getState());
Assert.assertEquals(VertexTerminationCause.ROOT_INPUT_INIT_FAILURE, v1.getTerminationCause());
}
+ @SuppressWarnings("unchecked")
+ @Test(timeout = 5000)
+ public void testExceptionFromII_InitFailedAfterInitialized() throws AMUserCodeException {
+ useCustomInitializer = true;
+ customInitializer = new EventHandlingRootInputInitializer(null, IIExceptionLocation.Initialize2);
+ EventHandlingRootInputInitializer initializer =
+ (EventHandlingRootInputInitializer) customInitializer;
+ setupPreDagCreation();
+ dagPlan = createDAGPlanWithIIException();
+ setupPostDagCreation();
+
+ VertexImplWithRunningInputInitializer v1 =
+ (VertexImplWithRunningInputInitializer) vertices.get("vertex1");
+ // INIT_SUCCEEDED followed by INIT_FAILURE
+ initVertex(v1);
+ dispatcher.getEventHandler().handle(new VertexEventRootInputInitialized(
+ v1.getVertexId(), "input1", null));
+ dispatcher.await();
+
+ String diagnostics = StringUtils.join(v1.getDiagnostics(), ",");
+ assertTrue(diagnostics.contains(IIExceptionLocation.Initialize2.name()));
+ Assert.assertEquals(VertexState.FAILED, v1.getState());
+ Assert.assertEquals(VertexTerminationCause.ROOT_INPUT_INIT_FAILURE, v1.getTerminationCause());
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test(timeout = 5000)
+ public void testExceptionFromII_InitFailedAfterRunning() throws AMUserCodeException {
+ useCustomInitializer = true;
+ customInitializer = new EventHandlingRootInputInitializer(null, IIExceptionLocation.Initialize2);
+ EventHandlingRootInputInitializer initializer =
+ (EventHandlingRootInputInitializer) customInitializer;
+ setupPreDagCreation();
+ dagPlan = createDAGPlanWithIIException();
+ setupPostDagCreation();
+
+ VertexImplWithRunningInputInitializer v1 =
+ (VertexImplWithRunningInputInitializer) vertices.get("vertex1");
+ initVertex(v1);
+ dispatcher.getEventHandler().handle(new VertexEventRootInputInitialized(
+ v1.getVertexId(), "input1", null));
+ dispatcher.getEventHandler().handle(new VertexEvent(v1.getVertexId(),
+ VertexEventType.V_START));
+ dispatcher.await();
+
+ String diagnostics = StringUtils.join(v1.getDiagnostics(), ",");
+ assertTrue(diagnostics.contains(IIExceptionLocation.Initialize2.name()));
+ Assert.assertEquals(VertexState.FAILED, v1.getState());
+ Assert.assertEquals(VertexTerminationCause.ROOT_INPUT_INIT_FAILURE, v1.getTerminationCause());
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test(timeout = 5000)
+ public void testExceptionFromII_HandleInputInitializerEvent() throws AMUserCodeException, InterruptedException {
+ useCustomInitializer = true;
+ customInitializer = new EventHandlingRootInputInitializer(null, IIExceptionLocation.HandleInputInitializerEvent);
+ EventHandlingRootInputInitializer initializer =
+ (EventHandlingRootInputInitializer) customInitializer;
+ setupPreDagCreation();
+ dagPlan = createDAGPlanWithRunningInitializer();
+ setupPostDagCreation();
+
+ VertexImplWithRunningInputInitializer v1 =
+ (VertexImplWithRunningInputInitializer) vertices.get("vertex1");
+ VertexImplWithRunningInputInitializer v2 =
+ (VertexImplWithRunningInputInitializer) vertices.get("vertex2");
+
+ initVertex(v1);
+ startVertex(v1);
+ Assert.assertEquals(VertexState.RUNNING, v1.getState());
+ Assert.assertEquals(VertexState.INITIALIZING, v2.getState());
+ dispatcher.await();
+
+ // Wait for the initializer to be invoked - which may be a separate thread.
+ while (!initializer.initStarted.get()) {
+ Thread.sleep(10);
+ }
+ Assert.assertFalse(initializer.eventReceived.get());
+ Assert.assertFalse(initializer.initComplete.get());
+
+ // Signal the initializer by sending an event - via vertex1
+ InputInitializerEvent event = InputInitializerEvent.create("vertex2", "input1", null);
+ // Create taskId and taskAttemptId for the single task that exists in vertex1
+ TezTaskID t0_v1 = TezTaskID.getInstance(v1.getVertexId(), 0);
+ TezTaskAttemptID ta0_t0_v1 = TezTaskAttemptID.getInstance(t0_v1, 0);
+ TezEvent tezEvent = new TezEvent(event,
+ new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex1", "vertex2", ta0_t0_v1));
+
+ // at least one task attempt is succeed, otherwise input initialize events won't been handled.
+ dispatcher.getEventHandler().handle(new TaskEvent(t0_v1, TaskEventType.T_ATTEMPT_LAUNCHED));
+ dispatcher.getEventHandler().handle(new TaskEventTAUpdate(ta0_t0_v1, TaskEventType.T_ATTEMPT_SUCCEEDED));
+ dispatcher.getEventHandler()
+ .handle(new VertexEventRouteEvent(v1.getVertexId(), Collections.singletonList(tezEvent)));
+ dispatcher.await();
+
+ // it would cause v2 fail as its II throw exception in handleInputInitializerEvent
+ String diagnostics = StringUtils.join(v2.getDiagnostics(), ",");
+ assertTrue(diagnostics.contains(IIExceptionLocation.HandleInputInitializerEvent.name()));
+ Assert.assertEquals(VertexState.FAILED, v2.getState());
+ Assert.assertEquals(VertexTerminationCause.ROOT_INPUT_INIT_FAILURE, v2.getTerminationCause());
+ }
+
+ @Test(timeout = 5000)
+ public void testExceptionFromII_OnVertexStateUpdated() throws AMUserCodeException, InterruptedException {
+ useCustomInitializer = true;
+ customInitializer = new EventHandlingRootInputInitializer(null, IIExceptionLocation.OnVertexStateUpdated);
+ EventHandlingRootInputInitializer initializer =
+ (EventHandlingRootInputInitializer) customInitializer;
+ setupPreDagCreation();
+ dagPlan = createDAGPlanWithRunningInitializer();
+ setupPostDagCreation();
+
+ VertexImplWithRunningInputInitializer v1 =
+ (VertexImplWithRunningInputInitializer) vertices.get("vertex1");
+ VertexImplWithRunningInputInitializer v2 =
+ (VertexImplWithRunningInputInitializer) vertices.get("vertex2");
+
+ initVertex(v1);
+ startVertex(v1); // v2 would get the state update from v1
+ Assert.assertEquals(VertexState.RUNNING, v1.getState());
+ Assert.assertEquals(VertexState.FAILED, v2.getState());
+ String diagnostics = StringUtils.join(v2.getDiagnostics(), ",");
+ assertTrue(diagnostics.contains(IIExceptionLocation.OnVertexStateUpdated.name()));
+ Assert.assertEquals(VertexTerminationCause.ROOT_INPUT_INIT_FAILURE, v2.getTerminationCause());
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test(timeout = 5000)
+ public void testExceptionFromII_InitSucceededAfterInitFailure() throws AMUserCodeException, InterruptedException {
+ useCustomInitializer = true;
+ customInitializer = new EventHandlingRootInputInitializer(null, IIExceptionLocation.OnVertexStateUpdated);
+ EventHandlingRootInputInitializer initializer =
+ (EventHandlingRootInputInitializer) customInitializer;
+ setupPreDagCreation();
+ dagPlan = createDAGPlanWithRunningInitializer();
+ setupPostDagCreation();
+
+ VertexImplWithRunningInputInitializer v1 =
+ (VertexImplWithRunningInputInitializer) vertices.get("vertex1");
+ VertexImplWithRunningInputInitializer v2 =
+ (VertexImplWithRunningInputInitializer) vertices.get("vertex2");
+
+ initVertex(v1);
+ startVertex(v1); // v2 would get the state update from v1
+ // it should be OK receive INIT_SUCCEEDED event after INIT_FAILED event
+ dispatcher.getEventHandler().handle(new VertexEventRootInputInitialized(
+ v2.getVertexId(), "input1", null));
+
+ Assert.assertEquals(VertexState.RUNNING, v1.getState());
+ Assert.assertEquals(VertexState.FAILED, v2.getState());
+ String diagnostics = StringUtils.join(v2.getDiagnostics(), ",");
+ assertTrue(diagnostics.contains(IIExceptionLocation.OnVertexStateUpdated.name()));
+ Assert.assertEquals(VertexTerminationCause.ROOT_INPUT_INIT_FAILURE, v2.getTerminationCause());
+ }
@InterfaceAudience.Private
public static class RootInputSpecUpdaterVertexManager extends VertexManagerPlugin {
@@ -5119,6 +5276,9 @@ public class TestVertexImpl {
public static enum IIExceptionLocation {
Initialize,
+ Initialize2, // for test case that InputInitFailed after InputInitSucceeded
+ HandleInputInitializerEvent,
+ OnVertexStateUpdated
}
@InterfaceAudience.Private
@@ -5154,6 +5314,16 @@ public class TestVertexImpl {
if (exLocation == IIExceptionLocation.Initialize) {
throw new Exception(exLocation.name());
}
+ if (exLocation == IIExceptionLocation.Initialize2) {
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ // InputInitializerManager is been shutdown if Initialized succeeded,
+ // catch the exception and throw the exception to simulate the case that
+ // init failure after init succeeded
+ throw new Exception(exLocation.name());
+ }
+ }
context.registerForVertexStateUpdates("vertex1", null);
initStarted.set(true);
lock.lock();
@@ -5175,6 +5345,9 @@ public class TestVertexImpl {
@Override
public void handleInputInitializerEvent(List<InputInitializerEvent> events) throws
Exception {
+ if (exLocation == IIExceptionLocation.HandleInputInitializerEvent) {
+ throw new Exception(exLocation.name());
+ }
initializerEvents.addAll(events);
if (initializerEvents.size() == numExpectedEvents) {
eventReceived.set(true);
@@ -5197,6 +5370,9 @@ public class TestVertexImpl {
}
public void onVertexStateUpdated(VertexStateUpdate stateUpdate) {
+ if (exLocation == IIExceptionLocation.OnVertexStateUpdated) {
+ throw new RuntimeException(exLocation.name());
+ }
stateUpdates.add(stateUpdate);
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/4ec29425/tez-tests/src/test/java/org/apache/tez/test/TestExceptionPropagation.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestExceptionPropagation.java b/tez-tests/src/test/java/org/apache/tez/test/TestExceptionPropagation.java
index eef6ab3..0175d7b 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/TestExceptionPropagation.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/TestExceptionPropagation.java
@@ -48,6 +48,7 @@ import org.apache.tez.dag.api.Edge;
import org.apache.tez.dag.api.EdgeManagerPluginContext;
import org.apache.tez.dag.api.EdgeManagerPluginDescriptor;
import org.apache.tez.dag.api.EdgeProperty;
+import org.apache.tez.dag.api.EdgeProperty.DataMovementType;
import org.apache.tez.dag.api.InputDescriptor;
import org.apache.tez.dag.api.InputInitializerDescriptor;
import org.apache.tez.dag.api.OutputDescriptor;
@@ -62,9 +63,10 @@ import org.apache.tez.dag.api.EdgeProperty.DataSourceType;
import org.apache.tez.dag.api.EdgeProperty.SchedulingType;
import org.apache.tez.dag.api.client.DAGClient;
import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.dag.api.event.VertexStateUpdate;
+import org.apache.tez.dag.app.dag.impl.OneToOneEdgeManager;
import org.apache.tez.dag.app.dag.impl.RootInputVertexManager;
-import org.apache.tez.dag.app.dag.impl.ScatterGatherEdgeManager;
-import org.apache.tez.dag.library.vertexmanager.ShuffleVertexManager;
+import org.apache.tez.dag.library.vertexmanager.InputReadyVertexManager;
import org.apache.tez.runtime.api.AbstractLogicalIOProcessor;
import org.apache.tez.runtime.api.AbstractLogicalInput;
import org.apache.tez.runtime.api.AbstractLogicalOutput;
@@ -84,8 +86,8 @@ import org.apache.tez.runtime.api.events.InputInitializerEvent;
import org.apache.tez.runtime.api.events.InputReadErrorEvent;
import org.apache.tez.runtime.api.events.VertexManagerEvent;
import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
-import org.apache.tez.runtime.library.processor.SleepProcessor;
-import org.apache.tez.runtime.library.processor.SleepProcessor.SleepProcessorConfig;
+import org.apache.tez.test.TestAMRecovery.DoNothingProcessor;
+import org.apache.tez.test.dag.MultiAttemptDAG.NoOpInput;
import org.junit.Test;
import com.google.common.collect.Lists;
@@ -210,7 +212,7 @@ public class TestExceptionPropagation {
* @throws Exception
*
*/
- @Test(timeout = 120000)
+ @Test(timeout = 180000)
public void testExceptionPropagationSession() throws Exception {
try {
startSessionClient();
@@ -309,6 +311,10 @@ public class TestExceptionPropagation {
EM_RouteInputErrorEventToSource,
// Not Supported yet
// EM_RouteInputSourceTaskFailedEventToDestination,
+
+ // II
+ II_Initialize, II_HandleInputInitializerEvents, II_OnVertexStateUpdated
+
}
/**
@@ -333,22 +339,31 @@ public class TestExceptionPropagation {
v1.setVertexManagerPlugin(RootInputVertexManagerWithException.getVMDesc(payload));
Vertex v2 =
- Vertex.create("v2",
- ProcessorDescriptor.create(SleepProcessor.class.getName())
- .setUserPayload(new SleepProcessorConfig(3).toUserPayload())
- , 1);
- v2.setVertexManagerPlugin(ShuffleVertexManagerWithException.getVMDesc(exLocation));
+ Vertex.create("v2", DoNothingProcessor.getProcDesc(), 1);
+ v2.addDataSource("input2",
+ DataSourceDescriptor.create(InputDescriptor.create(NoOpInput.class.getName()),
+ InputInitializerWithException2.getIIDesc(payload), null));
dag.addVertex(v1)
- .addVertex(v2)
- .addEdge(Edge.create(v1, v2, EdgeProperty.create(
- EdgeManagerPluginDescriptor.create(CustomEdgeManager.class.getName())
- .setUserPayload(payload),
- DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- OutputWithException.getOutputDesc(payload), InputWithException.getInputDesc(payload))));
+ .addVertex(v2);
+ if (exLocation.name().startsWith("EM_")) {
+ dag.addEdge(Edge.create(v1, v2, EdgeProperty.create(
+ EdgeManagerPluginDescriptor.create(CustomEdgeManager.class.getName())
+ .setUserPayload(payload),
+ DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
+ OutputWithException.getOutputDesc(payload), InputWithException.getInputDesc(payload))));
+ } else {
+ // set Customized VertexManager here, it can't been used for CustomEdge
+ v2.setVertexManagerPlugin(InputReadyVertexManagerWithException.getVMDesc(exLocation));
+ dag.addEdge(Edge.create(v1, v2, EdgeProperty.create(DataMovementType.ONE_TO_ONE,
+ DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
+ OutputWithException.getOutputDesc(payload), InputWithException.getInputDesc(payload))));
+ }
+
return dag;
}
+ // InputInitializer of vertex1
public static class InputInitializerWithException extends InputInitializer {
private ExceptionLocation exLocation;
@@ -380,7 +395,65 @@ public class TestExceptionPropagation {
}
}
- // input of vertex2
+ // InputInitializer of vertex2
+ public static class InputInitializerWithException2 extends InputInitializer {
+
+ private ExceptionLocation exLocation;
+ private Object condition = new Object();
+
+ public InputInitializerWithException2(
+ InputInitializerContext initializerContext) {
+ super(initializerContext);
+ this.exLocation =
+ ExceptionLocation.valueOf(new String(getContext().getUserPayload()
+ .deepCopyAsArray()));
+ }
+
+ @Override
+ public List<Event> initialize() throws Exception {
+ if (exLocation == ExceptionLocation.II_Initialize) {
+ throw new Exception(exLocation.name());
+ }
+ if (exLocation == ExceptionLocation.II_OnVertexStateUpdated) {
+ getContext().registerForVertexStateUpdates("v1", null);
+ }
+
+ if (exLocation == ExceptionLocation.II_HandleInputInitializerEvents
+ || exLocation == ExceptionLocation.II_OnVertexStateUpdated) {
+ // wait for handleInputInitializerEvent() and onVertexStateUpdated() is invoked
+ synchronized (condition) {
+ condition.wait();
+ }
+ }
+
+ return null;
+ }
+
+ @Override
+ public void handleInputInitializerEvent(List<InputInitializerEvent> events)
+ throws Exception {
+ if (exLocation == ExceptionLocation.II_HandleInputInitializerEvents) {
+ throw new RuntimeException(exLocation.name());
+ }
+ }
+
+ @Override
+ public void onVertexStateUpdated(VertexStateUpdate stateUpdate)
+ throws Exception {
+ if (exLocation == ExceptionLocation.II_OnVertexStateUpdated) {
+ throw new Exception(exLocation.name());
+ }
+ super.onVertexStateUpdated(stateUpdate);
+ }
+
+ public static InputInitializerDescriptor getIIDesc(UserPayload payload) {
+ return InputInitializerDescriptor.create(
+ InputInitializerWithException2.class.getName())
+ .setUserPayload(payload);
+ }
+ }
+
+ // Input of vertex2
public static class InputWithException extends AbstractLogicalInput {
private ExceptionLocation exLocation;
@@ -436,10 +509,12 @@ public class TestExceptionPropagation {
getContext().requestInitialMemory(0l, null); // mandatory call
if (this.exLocation == ExceptionLocation.INPUT_INITIALIZE) {
throw new Exception(this.exLocation.name());
- } else if (this.exLocation == ExceptionLocation.EM_RouteInputErrorEventToSource
- || this.exLocation == ExceptionLocation.EM_GetNumDestinationConsumerTasks) {
- Event errorEvent = InputReadErrorEvent.create("read error", 0, 0);
- return Lists.newArrayList(errorEvent);
+ } else if ( getContext().getSourceVertexName().equals("v1")) {
+ if (this.exLocation == ExceptionLocation.EM_RouteInputErrorEventToSource
+ || this.exLocation == ExceptionLocation.EM_GetNumDestinationConsumerTasks) {
+ Event errorEvent = InputReadErrorEvent.create("read error", 0, 0);
+ return Lists.newArrayList(errorEvent);
+ }
}
return null;
}
@@ -450,7 +525,7 @@ public class TestExceptionPropagation {
}
}
- // output of vertex1
+ // Output of vertex1
public static class OutputWithException extends AbstractLogicalOutput {
private ExceptionLocation exLocation;
@@ -497,8 +572,12 @@ public class TestExceptionPropagation {
List<Event> events = new ArrayList<Event>();
events.add(DataMovementEvent.create(0, ByteBuffer.wrap(new byte[0])));
return events;
- }
- else {
+ } else if (this.exLocation == ExceptionLocation.II_HandleInputInitializerEvents) {
+ // send InputInitliazer to InputInitializer of v2
+ List<Event> events = new ArrayList<Event>();
+ events.add(InputInitializerEvent.create("v2", "input2", ByteBuffer.wrap(new byte[0])));
+ return events;
+ } else {
return null;
}
}
@@ -576,6 +655,7 @@ public class TestExceptionPropagation {
}
}
+ // VertexManager of vertex1
public static class RootInputVertexManagerWithException extends RootInputVertexManager {
private ExceptionLocation exLocation;
@@ -618,12 +698,13 @@ public class TestExceptionPropagation {
}
}
- public static class ShuffleVertexManagerWithException extends ShuffleVertexManager {
+ // VertexManager of vertex2
+ public static class InputReadyVertexManagerWithException extends InputReadyVertexManager {
private ExceptionLocation exLocation;
private static final String Test_ExceptionLocation = "Test.ExceptionLocation";
- public ShuffleVertexManagerWithException(VertexManagerPluginContext context) {
+ public InputReadyVertexManagerWithException(VertexManagerPluginContext context) {
super(context);
}
@@ -666,12 +747,13 @@ public class TestExceptionPropagation {
Configuration conf = new Configuration();
conf.set(Test_ExceptionLocation, exLocation.name());
UserPayload payload = TezUtils.createUserPayloadFromConf(conf);
- return VertexManagerPluginDescriptor.create(ShuffleVertexManagerWithException.class.getName())
+ return VertexManagerPluginDescriptor.create(InputReadyVertexManagerWithException.class.getName())
.setUserPayload(payload);
}
}
- public static class CustomEdgeManager extends ScatterGatherEdgeManager {
+ // EdgeManager for edge linking vertex1 and vertex2
+ public static class CustomEdgeManager extends OneToOneEdgeManager {
private ExceptionLocation exLocation;
[50/50] [abbrv] tez git commit: Upmerge for final test of TEZ-8 branch
Posted by je...@apache.org.
Upmerge for final test of TEZ-8 branch
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/052f71cc
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/052f71cc
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/052f71cc
Branch: refs/heads/TEZ-8
Commit: 052f71cc7377b21e204aaa42cbafb4a25d1be1b5
Parents: 2ce8ebe 571cea4
Author: Jonathan Eagles <je...@gmail.com>
Authored: Mon Nov 17 13:21:40 2014 -0600
Committer: Jonathan Eagles <je...@gmail.com>
Committed: Mon Nov 17 13:21:40 2014 -0600
----------------------------------------------------------------------
.gitignore | 1 +
CHANGES.txt | 119 +-
docs/pom.xml | 14 +-
docs/src/site/markdown/index_0_5_0.md | 29 -
docs/src/site/markdown/install.md | 4 +
docs/src/site/markdown/releases/index_0_5_0.md | 29 +
docs/src/site/markdown/releases/index_0_5_1.md | 29 +
docs/src/site/site.xml | 3 +-
pom.xml | 3 +-
tez-api/pom.xml | 22 +
.../apache/tez/client/TezApiVersionInfo.java | 29 +
.../java/org/apache/tez/client/TezClient.java | 35 +-
.../org/apache/tez/client/TezClientUtils.java | 253 ++--
.../org/apache/tez/common/ATSConstants.java | 12 +
.../org/apache/tez/common/TezCommonUtils.java | 40 +-
.../java/org/apache/tez/common/TezUtils.java | 36 +
.../org/apache/tez/common/TezYARNUtils.java | 62 +-
.../java/org/apache/tez/common/VersionInfo.java | 105 ++
.../org/apache/tez/common/counters/Limits.java | 16 +-
.../apache/tez/common/security/ACLManager.java | 5 +-
.../common/security/JobTokenSecretManager.java | 40 +-
.../main/java/org/apache/tez/dag/api/DAG.java | 33 +-
.../apache/tez/dag/api/DagTypeConverters.java | 56 +-
.../apache/tez/dag/api/EdgeManagerPlugin.java | 22 +-
.../apache/tez/dag/api/EntityDescriptor.java | 6 +-
.../apache/tez/dag/api/TaskLocationHint.java | 71 +-
.../apache/tez/dag/api/TezConfiguration.java | 244 +++-
.../org/apache/tez/dag/api/TezConstants.java | 4 +
.../apache/tez/dag/api/VertexManagerPlugin.java | 36 +-
.../tez/dag/api/VertexManagerPluginContext.java | 53 +-
.../tez/dag/api/client/DAGClientImpl.java | 12 -
.../org/apache/tez/dag/api/client/Progress.java | 22 +-
.../apache/tez/dag/api/event/VertexState.java | 17 +-
.../tez/runtime/api/InputInitializer.java | 7 +-
.../api/events/InputInitializerEvent.java | 6 +
tez-api/src/main/proto/DAGApiRecords.proto | 11 +-
.../resources/tez-api-version-info.properties | 22 +
.../org/apache/tez/client/TestTezClient.java | 65 +-
.../apache/tez/client/TestTezClientUtils.java | 177 ++-
.../apache/tez/common/TestTezCommonUtils.java | 9 +-
.../org/apache/tez/common/TestTezYARNUtils.java | 60 +
.../org/apache/tez/common/TestVersionInfo.java | 66 +
.../org/apache/tez/dag/api/TestDAGPlan.java | 18 +-
.../tez/dag/api/TestDagTypeConverters.java | 4 +-
.../tez/dag/api/TestTaskLocationHint.java | 62 +
.../resources/test1-version-info.properties | 22 +
.../resources/test2-version-info.properties | 21 +
.../resources/test3-version-info.properties | 22 +
.../tez/common/EnvironmentUpdateUtils.java | 4 +-
.../org/apache/tez/common/TezUtilsInternal.java | 28 +
.../tez/common/TestEnvironmentUpdateUtils.java | 64 +-
.../org/apache/tez/common/TestTezUtils.java | 57 +-
tez-dag/findbugs-exclude.xml | 2 +-
tez-dag/pom.xml | 22 +
.../java/org/apache/tez/client/LocalClient.java | 4 +-
.../tez/dag/api/client/ProgressBuilder.java | 8 +
.../java/org/apache/tez/dag/app/AppContext.java | 4 +-
.../org/apache/tez/dag/app/DAGAppMaster.java | 199 ++-
.../apache/tez/dag/app/TezDagVersionInfo.java | 28 +
.../java/org/apache/tez/dag/app/dag/DAG.java | 2 +
.../app/dag/RootInputInitializerManager.java | 59 +-
.../java/org/apache/tez/dag/app/dag/Task.java | 20 +-
.../org/apache/tez/dag/app/dag/TaskAttempt.java | 2 -
.../dag/app/dag/TaskAttemptStateInternal.java | 1 -
.../tez/dag/app/dag/TaskTerminationCause.java | 7 +-
.../java/org/apache/tez/dag/app/dag/Vertex.java | 20 +-
.../tez/dag/app/dag/VertexTerminationCause.java | 5 +
...DAGAppMasterEventSchedulingServiceError.java | 34 +
.../app/dag/event/DAGAppMasterEventType.java | 3 +-
.../app/dag/event/TaskAttemptEventSchedule.java | 19 +-
.../dag/app/dag/event/TaskAttemptEventType.java | 1 -
.../tez/dag/app/dag/event/TaskEventType.java | 1 -
.../event/VertexEventManagerUserCodeError.java | 36 +
.../dag/event/VertexEventRootInputFailed.java | 7 +-
.../tez/dag/app/dag/event/VertexEventType.java | 1 +
.../dag/app/dag/impl/AMUserCodeException.java | 56 +
.../apache/tez/dag/app/dag/impl/DAGImpl.java | 133 +-
.../tez/dag/app/dag/impl/DAGSchedulerMRR.java | 334 -----
.../app/dag/impl/DAGSchedulerNaturalOrder.java | 19 +-
.../DAGSchedulerNaturalOrderControlled.java | 256 ++++
.../org/apache/tez/dag/app/dag/impl/Edge.java | 121 +-
.../dag/impl/ImmediateStartVertexManager.java | 93 +-
.../tez/dag/app/dag/impl/TaskAttemptImpl.java | 114 +-
.../apache/tez/dag/app/dag/impl/TaskImpl.java | 183 +--
.../apache/tez/dag/app/dag/impl/VertexImpl.java | 1034 ++++++++++-----
.../tez/dag/app/dag/impl/VertexManager.java | 220 +++-
.../dag/app/launcher/ContainerLauncherImpl.java | 17 +-
.../app/rm/AMSchedulerEventTALaunchRequest.java | 7 +-
.../dag/app/rm/TaskSchedulerEventHandler.java | 55 +-
.../dag/app/rm/YarnTaskSchedulerService.java | 193 ++-
.../tez/dag/app/rm/node/AMNodeEventType.java | 11 +-
.../apache/tez/dag/app/rm/node/AMNodeImpl.java | 38 +-
.../apache/tez/dag/app/rm/node/AMNodeMap.java | 225 ----
.../tez/dag/app/rm/node/AMNodeTracker.java | 211 +++
.../tez/dag/history/HistoryEventType.java | 1 +
.../dag/history/events/AppLaunchedEvent.java | 104 ++
.../dag/history/events/DAGFinishedEvent.java | 11 +-
.../dag/history/events/DAGInitializedEvent.java | 10 +-
.../dag/history/events/DAGSubmittedEvent.java | 1 +
.../history/events/TaskAttemptStartedEvent.java | 9 +-
.../dag/history/events/TaskFinishedEvent.java | 1 +
.../dag/history/events/VertexFinishedEvent.java | 12 +-
.../events/VertexParallelismUpdatedEvent.java | 17 +-
.../tez/dag/history/logging/EntityTypes.java | 1 +
.../impl/HistoryEventJsonConversion.java | 118 +-
.../apache/tez/dag/history/utils/DAGUtils.java | 111 +-
.../utils/Simple2LevelVersionComparator.java | 105 ++
.../org/apache/tez/runtime/task/TezChild.java | 2 +-
tez-dag/src/main/proto/HistoryEvents.proto | 2 +
.../resources/tez-container-log4j.properties | 2 +-
.../resources/tez-dag-version-info.properties | 22 +
.../org/apache/tez/client/TestLocalMode.java | 2 +-
.../apache/tez/dag/app/MockDAGAppMaster.java | 31 +-
.../tez/dag/app/TestMockDAGAppMaster.java | 27 +-
.../org/apache/tez/dag/app/TestPreemption.java | 6 +-
.../apache/tez/dag/app/TestRecoveryParser.java | 6 +-
.../tez/dag/app/dag/impl/TestDAGImpl.java | 462 ++++++-
.../tez/dag/app/dag/impl/TestDAGRecovery.java | 19 +-
.../tez/dag/app/dag/impl/TestDAGScheduler.java | 137 +-
.../TestDAGSchedulerNaturalOrderControlled.java | 374 ++++++
.../apache/tez/dag/app/dag/impl/TestEdge.java | 5 +-
.../impl/TestImmediateStartVertexManager.java | 131 ++
.../tez/dag/app/dag/impl/TestTaskAttempt.java | 177 +--
.../app/dag/impl/TestTaskAttemptRecovery.java | 2 +-
.../tez/dag/app/dag/impl/TestTaskImpl.java | 65 +-
.../tez/dag/app/dag/impl/TestTaskRecovery.java | 181 ++-
.../tez/dag/app/dag/impl/TestVertexImpl.java | 1200 ++++++++++++++----
.../tez/dag/app/dag/impl/TestVertexManager.java | 105 +-
.../dag/app/dag/impl/TestVertexRecovery.java | 137 +-
.../tez/dag/app/rm/TestContainerReuse.java | 32 +-
.../tez/dag/app/rm/TestTaskScheduler.java | 86 +-
.../app/rm/TestTaskSchedulerEventHandler.java | 62 +-
.../tez/dag/app/rm/node/TestAMNodeMap.java | 268 ----
.../tez/dag/app/rm/node/TestAMNodeTracker.java | 339 +++++
.../TestHistoryEventsProtoConversion.java | 41 +-
.../impl/TestHistoryEventJsonConversion.java | 64 +-
.../tez/dag/history/utils/TestDAGUtils.java | 23 +-
.../TestSimple2LevelVersionComparator.java | 66 +
.../org/apache/tez/examples/ExampleDriver.java | 61 -
.../apache/tez/examples/HashJoinExample.java | 4 +-
.../org/apache/tez/examples/JoinValidate.java | 2 +-
.../apache/tez/examples/OrderedWordCount.java | 4 +-
.../tez/examples/SortMergeJoinExample.java | 18 +-
.../java/org/apache/tez/examples/WordCount.java | 2 +-
.../mapred/split/TezMapredSplitsGrouper.java | 30 +-
.../split/TezMapReduceSplitsGrouper.java | 41 +-
.../apache/tez/mapreduce/client/YARNRunner.java | 14 +
.../tez/mapreduce/hadoop/DeprecatedKeys.java | 12 +-
.../org/apache/tez/mapreduce/input/MRInput.java | 26 +-
.../tez/mapreduce/input/MultiMRInput.java | 5 +-
.../apache/tez/mapreduce/output/MROutput.java | 6 +
.../hadoop/mapred/split/TestGroupedSplits.java | 142 +++
.../tez/mapreduce/input/TestMultiMRInput.java | 10 +-
.../tez/mapreduce/processor/MapUtils.java | 2 +-
.../processor/reduce/TestReduceProcessor.java | 2 +-
tez-plugins/pom.xml | 28 +-
.../tez-mbeans-resource-calculator/pom.xml | 60 +
.../tez/util/TezMxBeanResourceCalculator.java | 74 ++
.../util/TestTezMxBeanResourceCalculator.java | 74 ++
.../logging/ats/ATSHistoryLoggingService.java | 158 ++-
.../ats/HistoryEventTimelineConversion.java | 131 +-
.../ats/TestATSHistoryLoggingService.java | 47 +-
.../ats/TestHistoryEventTimelineConversion.java | 455 ++++++-
.../src/test/resources/log4j.properties | 19 +
.../runtime/api/impl/TezInputContextImpl.java | 2 +-
.../api/impl/TezMergedInputContextImpl.java | 2 +-
.../runtime/api/impl/TezOutputContextImpl.java | 2 +-
.../api/impl/TezProcessorContextImpl.java | 2 +-
.../common/resources/MemoryDistributor.java | 8 +-
.../common/resources/ScalingAllocator.java | 4 +-
.../TestLogicalIOProcessorRuntimeTask.java | 2 +-
.../common/resources/TestMemoryDistributor.java | 8 +-
.../java/org/apache/hadoop/io/FileChunk.java | 42 +-
.../vertexmanager/InputReadyVertexManager.java | 11 +-
.../vertexmanager/ShuffleVertexManager.java | 217 +++-
.../library/api/TezRuntimeConfiguration.java | 8 +
.../common/readers/UnorderedKVReader.java | 10 +-
.../common/security/SecureShuffleUtils.java | 36 +-
.../common/shuffle/DiskFetchedInput.java | 121 ++
.../library/common/shuffle/FetchResult.java | 70 +
.../library/common/shuffle/FetchedInput.java | 152 +++
.../common/shuffle/FetchedInputAllocator.java | 34 +
.../common/shuffle/FetchedInputCallback.java | 29 +
.../runtime/library/common/shuffle/Fetcher.java | 970 ++++++++++++++
.../library/common/shuffle/FetcherCallback.java | 33 +
.../library/common/shuffle/HttpConnection.java | 426 +++++++
.../library/common/shuffle/InputHost.java | 133 ++
.../common/shuffle/LocalDiskFetchedInput.java | 119 ++
.../common/shuffle/MemoryFetchedInput.java | 90 ++
.../common/shuffle/ShuffleEventHandler.java | 28 +
.../library/common/shuffle/ShuffleUtils.java | 240 ++++
.../common/shuffle/impl/ExceptionReporter.java | 25 -
.../library/common/shuffle/impl/Fetcher.java | 595 ---------
.../common/shuffle/impl/InMemoryReader.java | 162 ---
.../common/shuffle/impl/InMemoryWriter.java | 61 -
.../library/common/shuffle/impl/MapHost.java | 124 --
.../library/common/shuffle/impl/MapOutput.java | 234 ----
.../common/shuffle/impl/MergeManager.java | 930 --------------
.../common/shuffle/impl/MergeThread.java | 108 --
.../library/common/shuffle/impl/Shuffle.java | 484 -------
.../shuffle/impl/ShuffleClientMetrics.java | 92 --
.../common/shuffle/impl/ShuffleHeader.java | 94 --
.../shuffle/impl/ShuffleInputEventHandler.java | 134 --
.../impl/ShuffleInputEventHandlerImpl.java | 169 +++
.../common/shuffle/impl/ShuffleManager.java | 802 ++++++++++++
.../common/shuffle/impl/ShuffleScheduler.java | 620 ---------
.../impl/SimpleFetchedInputAllocator.java | 200 +++
.../orderedgrouped/ExceptionReporter.java | 25 +
.../orderedgrouped/FetcherOrderedGrouped.java | 703 ++++++++++
.../shuffle/orderedgrouped/InMemoryReader.java | 162 +++
.../shuffle/orderedgrouped/InMemoryWriter.java | 61 +
.../common/shuffle/orderedgrouped/MapHost.java | 124 ++
.../shuffle/orderedgrouped/MapOutput.java | 237 ++++
.../shuffle/orderedgrouped/MergeManager.java | 945 ++++++++++++++
.../shuffle/orderedgrouped/MergeThread.java | 109 ++
.../common/shuffle/orderedgrouped/Shuffle.java | 483 +++++++
.../orderedgrouped/ShuffleClientMetrics.java | 92 ++
.../shuffle/orderedgrouped/ShuffleHeader.java | 94 ++
.../ShuffleInputEventHandlerOrderedGrouped.java | 137 ++
.../orderedgrouped/ShuffleScheduler.java | 626 +++++++++
.../shuffle/orderedgrouped/package-info.java | 22 +
.../library/common/shuffle/package-info.java | 22 +
.../common/sort/impl/ExternalSorter.java | 2 +-
.../runtime/library/common/sort/impl/IFile.java | 10 +-
.../common/sort/impl/PipelinedSorter.java | 5 +-
.../common/sort/impl/dflt/DefaultSorter.java | 17 +-
.../local/output/TezLocalTaskOutputFiles.java | 10 +-
.../common/task/local/output/TezTaskOutput.java | 7 +
.../task/local/output/TezTaskOutputFiles.java | 7 +-
.../writers/UnorderedPartitionedKVWriter.java | 2 +-
.../HadoopKeyValuesBasedBaseEdgeConfig.java | 13 +
.../conf/OrderedGroupedKVInputConfig.java | 15 +-
.../conf/OrderedPartitionedKVEdgeConfig.java | 19 +
.../conf/OrderedPartitionedKVOutputConfig.java | 15 +-
.../library/conf/UnorderedKVEdgeConfig.java | 20 +
.../library/conf/UnorderedKVInputConfig.java | 15 +-
.../library/conf/UnorderedKVOutputConfig.java | 15 +-
.../conf/UnorderedPartitionedKVEdgeConfig.java | 19 +
.../UnorderedPartitionedKVOutputConfig.java | 14 +-
.../apache/tez/runtime/library/conf/Utils.java | 45 +
.../exceptions/FetcherReadTimeoutException.java | 49 +
.../library/input/OrderedGroupedKVInput.java | 11 +-
.../runtime/library/input/UnorderedKVInput.java | 17 +-
.../output/OrderedPartitionedKVOutput.java | 11 +-
.../library/output/UnorderedKVOutput.java | 11 +-
.../output/UnorderedPartitionedKVOutput.java | 9 +-
.../WeightedScalingMemoryDistributor.java | 10 +-
.../shuffle/common/DiskFetchedInput.java | 121 --
.../library/shuffle/common/FetchResult.java | 70 -
.../library/shuffle/common/FetchedInput.java | 152 ---
.../shuffle/common/FetchedInputAllocator.java | 34 -
.../shuffle/common/FetchedInputCallback.java | 29 -
.../runtime/library/shuffle/common/Fetcher.java | 856 -------------
.../library/shuffle/common/FetcherCallback.java | 33 -
.../library/shuffle/common/HttpConnection.java | 414 ------
.../library/shuffle/common/InputHost.java | 133 --
.../shuffle/common/LocalDiskFetchedInput.java | 119 --
.../shuffle/common/MemoryFetchedInput.java | 90 --
.../shuffle/common/ShuffleEventHandler.java | 28 -
.../library/shuffle/common/ShuffleUtils.java | 237 ----
.../impl/ShuffleInputEventHandlerImpl.java | 163 ---
.../shuffle/common/impl/ShuffleManager.java | 804 ------------
.../impl/SimpleFetchedInputAllocator.java | 200 ---
.../shuffle/common/impl/package-info.java | 22 -
.../library/shuffle/common/package-info.java | 22 -
.../TestInputReadyVertexManager.java | 45 +-
.../vertexmanager/TestShuffleVertexManager.java | 563 ++++++--
.../TestWeightedScalingMemoryDistributor.java | 18 +-
.../library/common/TestValuesIterator.java | 6 +-
.../library/common/shuffle/TestFetcher.java | 204 +++
.../common/shuffle/impl/TestFetcher.java | 161 ---
.../impl/TestShuffleInputEventHandler.java | 173 ---
.../impl/TestShuffleInputEventHandlerImpl.java | 162 +++
.../impl/TestSimpleFetchedInputAllocator.java | 88 ++
.../shuffle/orderedgrouped/TestFetcher.java | 298 +++++
.../orderedgrouped/TestMergeManager.java | 237 ++++
.../TestShuffleInputEventHandler.java | 172 +++
.../library/common/sort/impl/TestIFile.java | 5 +-
.../TestUnorderedPartitionedKVWriter.java | 2 +-
.../TestOrderedPartitionedKVEdgeConfig.java | 45 +-
.../library/conf/TestUnorderedKVEdgeConfig.java | 48 +-
.../TestUnorderedPartitionedKVEdgeConfig.java | 44 +-
.../library/output/TestOnFileSortedOutput.java | 2 +-
.../output/TestOnFileUnorderedKVOutput.java | 2 +-
.../library/shuffle/common/TestFetcher.java | 199 ---
.../impl/TestShuffleInputEventHandlerImpl.java | 163 ---
.../impl/TestSimpleFetchedInputAllocator.java | 88 --
.../examples/BroadcastAndOneToOneExample.java | 4 +-
.../mapreduce/examples/BroadcastLoadGen.java | 148 +++
.../tez/mapreduce/examples/ExampleDriver.java | 11 +-
.../org/apache/tez/mapreduce/examples/Join.java | 7 +-
.../tez/mapreduce/examples/MapredWordCount.java | 7 +-
.../mapreduce/examples/RandomTextWriter.java | 4 +-
.../tez/mapreduce/examples/RandomWriter.java | 4 +-
.../org/apache/tez/mapreduce/examples/Sort.java | 4 +-
.../examples/TestOrderedWordCount.java | 1 +
.../apache/tez/mapreduce/TestMRRJobsDAGApi.java | 10 +-
.../org/apache/tez/test/TestAMRecovery.java | 5 +-
.../tez/test/TestExceptionPropagation.java | 342 ++++-
.../org/apache/tez/test/TestFaultTolerance.java | 17 +-
.../org/apache/tez/test/TestSecureShuffle.java | 2 +-
.../java/org/apache/tez/test/TestTezJobs.java | 138 ++
tez-tools/swimlanes/amlogparser.py | 2 +-
tez-tools/tez-tfile-parser/README.txt | 24 +
tez-tools/tez-tfile-parser/pom.xml | 150 +++
.../java/org/apache/tez/tools/TFileLoader.java | 184 +++
.../org/apache/tez/tools/TFileRecordReader.java | 107 ++
tez-ui/src/main/webapp/bower.json | 1 -
308 files changed, 19974 insertions(+), 11289 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/052f71cc/.gitignore
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/052f71cc/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index d2d7a21,bb31375..69a6e94
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -6,23 -6,41 +6,54 @@@ Release 0.6.0: Unrelease
INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-1634. Fix compressed IFile shuffle errors
+ TEZ-1615. Skeleton framework for Tez UI
+ TEZ-1604. Task View for Tez UI
+ TEZ-1603. Vertex View for Tez UI.
+ TEZ-1720. Allow filters in all tables and also to pass in filters using url params.
+ TEZ-1708. Make UI part of TEZ build process.
+ TEZ-1617. Shim layer for Tez UI for use within Ambari.
+ TEZ-1741. App view.
+ TEZ-1751. Log view & download links in task and task attempt view.
+ TEZ-1753. Queue in dags view.
+ TEZ-1765. Allow dropdown lists in table filters.
+ TEZ-1606. Counters View for DAG, Vertex, and Task.
+ TEZ-1768. follow up jira to address minor issues in Tez-ui.
+ TEZ-1733. TezMerger should sort FileChunks on size when merging
+ TEZ-1738. Tez tfile parser for log parsing
+ TEZ-1627. Remove OUTPUT_CONSUMABLE and related Event in TaskAttemptImpl
+ TEZ-1736. Add support for Inputs/Outputs in runtime-library to generate history text data.
+ TEZ-1721. Update INSTALL instructions for clarifying tez client jars
+ compatibility with runtime tarball on HDFS.
+ TEZ-1690. TestMultiMRInput tests fail because of user collisions.
- Release 0.5.2: Unreleased
+ Release 0.5.3: Unreleased
+
+ ALL CHANGES:
+ TEZ-1758. TezClient should provide YARN diagnostics when the AM crashes
+ TEZ-1742. Improve response time of internal preemption
+ TEZ-1745. TestATSHistoryLoggingService::testATSHistoryLoggingServiceShutdown can be flaky.
+ TEZ-1747. Increase test timeout for TestSecureShuffle.
+ TEZ-1746. Flaky test in TestVertexImpl and TestExceptionPropagation.
+ TEZ-1749. Increase test timeout for TestLocalMode.testMultipleClientsWithSession
+ TEZ-1750. Add a DAGScheduler which schedules tasks only when sources have been scheduled.
+ TEZ-1761. TestRecoveryParser::testGetLastInProgressDAG fails in similar manner to TEZ-1686.
+ TEZ-1687. Use logIdentifier of Vertex for logging.
+ TEZ-1737. Should add taskNum in VertexFinishedEvent.
+ TEZ-1770. Handle ConnectExceptions correctly when establishing connections to an NM which may be down.
+ TEZ-1774. AppLaunched event for Timeline does not have start time set.
+ TEZ-1772. Failing tests post TEZ-1737.
+
+ Release 0.5.2: 2014-11-07
INCOMPATIBLE CHANGES
+ TEZ-1666. UserPayload should be null if the payload is not specified.
+ 0.5.1 client cannot talk to 0.5.2 AMs (TEZ-1666 and TEZ-1664).
+ context.getUserPayload can now return null, apps may need to add defensive code.
+ TEZ-1699. Vertex.setParallelism should throw an exception for invalid
+ invocations
+ TEZ-1700. Replace containerId from TaskLocationHint with [TaskIndex+Vertex]
+ based affinity
ALL CHANGES:
TEZ-1620. Wait for application finish before stopping MiniTezCluster
@@@ -50,9 -149,7 +162,10 @@@ ALL CHANGE
TEZ-1580. Change TestOrderedWordCount to optionally use MR configs.
TEZ-1524. Resolve user group information only if ACLs are enabled.
TEZ-1581. GroupByOrderByMRRTest no longer functional.
+ TEZ-1157. Optimize broadcast shuffle to download data only once per host.
+ TEZ-1594. Initial TezUI into TEZ-8 branch
+ TEZ-1595. Document timeline server setup for use with the Tez UI
+ TEZ-1157. Optimize broadcast shuffle to download data only once per host.
TEZ-1607. support mr envs in mrrsleep and testorderedwordcount
TEZ-1499. Add SortMergeJoinExample to tez-examples
TEZ-1613. Decrease running time for TestAMRecovery
http://git-wip-us.apache.org/repos/asf/tez/blob/052f71cc/docs/src/site/site.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/052f71cc/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/052f71cc/tez-ui/src/main/webapp/bower.json
----------------------------------------------------------------------
diff --cc tez-ui/src/main/webapp/bower.json
index efec4ec,0000000..18cae96
mode 100644,000000..100644
--- a/tez-ui/src/main/webapp/bower.json
+++ b/tez-ui/src/main/webapp/bower.json
@@@ -1,32 -1,0 +1,31 @@@
+{
+ "name": "tez-ui",
+ "version": "0.0.1",
+ "dependencies": {
+ "ember": "1.7.0",
+ "moment": ">=2.7.0",
+ "ember-data": "1.0.0-beta.10",
+ "ember-i18n": "1.6.*",
+ "bootstrap": "3.0.0",
+ "ember-json-mapper": "master",
+ "jquery-ui": ">=1.11",
+ "d3": "3.4.11",
+ "ember-addons.bs_for_ember": "~0.7.0",
+ "ember-table": "~0.2.2"
+ },
+ "devDependencies": {
+ "ember-mocha-adapter": "0.1.2",
+ "handlebars": "~2.0.0",
+ "jquery-ui": "~1.11.1",
+ "jquery-mousewheel": "~3.1.12",
+ "antiscroll": "*",
+ "font-awesome": "~4.2.0"
+ },
+ "resolutions": {
+ "handlebars": "~1.3.0",
+ "jquery-ui": ">=1.11",
+ "jquery-mousewheel": "~3.1.12",
+ "antiscroll": "fa3f81d3c0",
- "font-awesome": "~4.2.0",
+ "ember": "1.7.0"
+ }
+}
[42/50] [abbrv] tez git commit: TEZ-1737. Should add taskNum in
VertexFinishedEvent (zjffdu)
Posted by je...@apache.org.
TEZ-1737. Should add taskNum in VertexFinishedEvent (zjffdu)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/0ebfc1b1
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/0ebfc1b1
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/0ebfc1b1
Branch: refs/heads/TEZ-8
Commit: 0ebfc1b10b5000baadee930326b7f6b81516ea39
Parents: 103839b
Author: Jeff Zhang <zj...@apache.org>
Authored: Wed Nov 12 12:54:57 2014 +0800
Committer: Jeff Zhang <zj...@apache.org>
Committed: Wed Nov 12 12:54:57 2014 +0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/tez/dag/app/dag/impl/VertexImpl.java | 3 +-
.../dag/history/events/VertexFinishedEvent.java | 10 ++-
tez-dag/src/main/proto/HistoryEvents.proto | 1 +
.../dag/app/dag/impl/TestVertexRecovery.java | 89 +++++++++++++++++++-
.../TestHistoryEventsProtoConversion.java | 4 +-
.../impl/TestHistoryEventJsonConversion.java | 3 +-
.../ats/TestHistoryEventTimelineConversion.java | 4 +-
8 files changed, 105 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/0ebfc1b1/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 34e2c7a..7ad6903 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -22,6 +22,7 @@ ALL CHANGES:
TEZ-1750. Add a DAGScheduler which schedules tasks only when sources have been scheduled.
TEZ-1761. TestRecoveryParser::testGetLastInProgressDAG fails in similar manner to TEZ-1686.
TEZ-1687. Use logIdentifier of Vertex for logging.
+ TEZ-1737. Should add taskNum in VertexFinishedEvent.
Release 0.5.2: 2014-11-07
http://git-wip-us.apache.org/repos/asf/tez/blob/0ebfc1b1/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index 8130c18..d19c4cc 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -1135,6 +1135,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
} else {
vertexCompleteSeen = true;
}
+ numTasks = finishedEvent.getNumTasks();
recoveryCommitInProgress = false;
recoveredState = finishedEvent.getState();
diagnostics.add(finishedEvent.getDiagnostics());
@@ -1610,7 +1611,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
taskStats.put(ATSConstants.NUM_FAILED_TASKS_ATTEMPTS, failedTaskAttemptCount.get());
taskStats.put(ATSConstants.NUM_KILLED_TASKS_ATTEMPTS, killedTaskAttemptCount.get());
- VertexFinishedEvent finishEvt = new VertexFinishedEvent(vertexId, vertexName, initTimeRequested,
+ VertexFinishedEvent finishEvt = new VertexFinishedEvent(vertexId, vertexName, numTasks, initTimeRequested,
initedTime, startTimeRequested, startedTime, finishTime, finalState, diagnostics,
getAllCounters(), getVertexStats(), taskStats);
this.appContext.getHistoryHandler().handleCriticalEvent(
http://git-wip-us.apache.org/repos/asf/tez/blob/0ebfc1b1/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
index d9cafc7..d1d7eef 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
@@ -43,6 +43,7 @@ public class VertexFinishedEvent implements HistoryEvent, SummaryEvent {
private TezVertexID vertexID;
private String vertexName;
+ private int numTasks;
private long initRequestedTime;
private long initedTime;
private long startRequestedTime;
@@ -55,13 +56,14 @@ public class VertexFinishedEvent implements HistoryEvent, SummaryEvent {
private VertexStats vertexStats;
private Map<String, Integer> vertexTaskStats;
- public VertexFinishedEvent(TezVertexID vertexId, String vertexName, long initRequestedTime,
+ public VertexFinishedEvent(TezVertexID vertexId, String vertexName, int numTasks, long initRequestedTime,
long initedTime, long startRequestedTime, long startedTime,
long finishTime, VertexState state, String diagnostics,
TezCounters counters, VertexStats vertexStats,
Map<String, Integer> vertexTaskStats) {
this.vertexName = vertexName;
this.vertexID = vertexId;
+ this.numTasks = numTasks;
this.initRequestedTime = initRequestedTime;
this.initedTime = initedTime;
this.startRequestedTime = startRequestedTime;
@@ -182,12 +184,17 @@ public class VertexFinishedEvent implements HistoryEvent, SummaryEvent {
return vertexTaskStats;
}
+ public int getNumTasks() {
+ return numTasks;
+ }
+
@Override
public void toSummaryProtoStream(OutputStream outputStream) throws IOException {
VertexFinishStateProto finishStateProto =
VertexFinishStateProto.newBuilder()
.setState(state.ordinal())
.setVertexId(vertexID.toString())
+ .setNumTasks(numTasks)
.build();
SummaryEventProto.Builder builder = RecoveryProtos.SummaryEventProto.newBuilder()
@@ -205,6 +212,7 @@ public class VertexFinishedEvent implements HistoryEvent, SummaryEvent {
VertexFinishStateProto.parseFrom(proto.getEventPayload());
this.vertexID = TezVertexID.fromString(finishStateProto.getVertexId());
this.state = VertexState.values()[finishStateProto.getState()];
+ this.numTasks = finishStateProto.getNumTasks();
this.finishTime = proto.getTimestamp();
this.fromSummary = true;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/0ebfc1b1/tez-dag/src/main/proto/HistoryEvents.proto
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/proto/HistoryEvents.proto b/tez-dag/src/main/proto/HistoryEvents.proto
index 93f217f..5d18b0a 100644
--- a/tez-dag/src/main/proto/HistoryEvents.proto
+++ b/tez-dag/src/main/proto/HistoryEvents.proto
@@ -198,4 +198,5 @@ message SummaryEventProto {
message VertexFinishStateProto {
optional string vertex_id = 1;
optional int32 state = 2;
+ optional int32 num_tasks = 3;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/0ebfc1b1/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexRecovery.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexRecovery.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexRecovery.java
index 8fa574c..1b8ac2f 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexRecovery.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexRecovery.java
@@ -47,6 +47,7 @@ import org.apache.tez.dag.api.records.DAGProtos.PlanEdgeSchedulingType;
import org.apache.tez.dag.api.records.DAGProtos.PlanTaskConfiguration;
import org.apache.tez.dag.api.records.DAGProtos.PlanTaskLocationHint;
import org.apache.tez.dag.api.records.DAGProtos.PlanVertexType;
+import org.apache.tez.dag.api.records.DAGProtos.RootInputLeafOutputProto;
import org.apache.tez.dag.api.records.DAGProtos.TezEntityDescriptorProto;
import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
import org.apache.tez.dag.app.AppContext;
@@ -67,11 +68,14 @@ import org.apache.tez.dag.app.dag.event.VertexEventRecoverVertex;
import org.apache.tez.dag.app.dag.event.VertexEventType;
import org.apache.tez.dag.app.dag.impl.AMUserCodeException.Source;
import org.apache.tez.dag.app.dag.impl.TestVertexImpl.CountingOutputCommitter;
+import org.apache.tez.dag.history.HistoryEventType;
import org.apache.tez.dag.history.events.VertexRecoverableEventsGeneratedEvent;
import org.apache.tez.dag.history.events.VertexFinishedEvent;
import org.apache.tez.dag.history.events.VertexInitializedEvent;
import org.apache.tez.dag.history.events.VertexStartedEvent;
import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.recovery.records.RecoveryProtos.SummaryEventProto;
+import org.apache.tez.dag.recovery.records.RecoveryProtos.VertexFinishStateProto;
import org.apache.tez.runtime.api.OutputCommitter;
import org.apache.tez.runtime.api.events.InputDataInformationEvent;
import org.apache.tez.runtime.api.impl.EventMetaData;
@@ -215,6 +219,49 @@ public class TestVertexRecovery {
return dag;
}
+ private DAGPlan createDAGPlanSingleVertex() {
+ DAGPlan dag =
+ DAGPlan
+ .newBuilder()
+ .setName("testverteximpl")
+ .addVertex(
+ VertexPlan
+ .newBuilder()
+ .setName("vertex1")
+ .setType(PlanVertexType.NORMAL)
+ .addTaskLocationHint(
+ PlanTaskLocationHint.newBuilder().addHost("host1")
+ .addRack("rack1").build())
+ .setTaskConfig(
+ PlanTaskConfiguration.newBuilder().setNumTasks(-1)
+ .setVirtualCores(4).setMemoryMb(1024)
+ .setJavaOpts("").setTaskModule("x1.y1").build())
+ .addInputs(RootInputLeafOutputProto.newBuilder()
+ .setIODescriptor(
+ TezEntityDescriptorProto.newBuilder()
+ .setClassName("input").build())
+ .setName("inputx")
+ .setControllerDescriptor(
+ TezEntityDescriptorProto
+ .newBuilder()
+ .setClassName("inputinitlizer"))
+ .build())
+ .addOutputs(
+ DAGProtos.RootInputLeafOutputProto
+ .newBuilder()
+ .setIODescriptor(
+ TezEntityDescriptorProto.newBuilder()
+ .setClassName("output").build())
+ .setName("outputx")
+ .setControllerDescriptor(
+ TezEntityDescriptorProto
+ .newBuilder()
+ .setClassName(
+ CountingOutputCommitter.class.getName())))
+ .build()).build();
+ return dag;
+ }
+
class VertexEventHanlder implements EventHandler<VertexEvent> {
private List<VertexEvent> events = new ArrayList<VertexEvent>();
@@ -316,6 +363,44 @@ public class TestVertexRecovery {
}
/**
+ * vertex1(New) -> StartRecoveryTransition(SUCCEEDED)
+ * @throws IOException
+ */
+ @Test
+ public void testRecovery_Desired_SUCCEEDED_OnlySummaryLog() throws IOException {
+ DAGPlan dagPlan = createDAGPlanSingleVertex();
+ dag =
+ new DAGImpl(dagId, new Configuration(), dagPlan,
+ dispatcher.getEventHandler(), mock(TaskAttemptListener.class),
+ new Credentials(), new SystemClock(), user,
+ mock(TaskHeartbeatHandler.class), mockAppContext);
+ when(mockAppContext.getCurrentDAG()).thenReturn(dag);
+ dag.handle(new DAGEvent(dagId, DAGEventType.DAG_INIT));
+
+ VertexImpl vertex1 = (VertexImpl) dag.getVertex("vertex1");
+ VertexFinishedEvent vertexFinishEvent = new VertexFinishedEvent();
+ vertexFinishEvent.fromSummaryProtoStream(SummaryEventProto.newBuilder()
+ .setDagId(dag.getID().toString())
+ .setEventType(HistoryEventType.VERTEX_FINISHED.ordinal())
+ .setTimestamp(100L)
+ .setEventPayload(VertexFinishStateProto.newBuilder()
+ .setNumTasks(2)
+ .setState(VertexState.SUCCEEDED.ordinal())
+ .setVertexId(vertex1.getVertexId().toString()).build().toByteString())
+ .build());
+ VertexState recoveredState = vertex1.restoreFromEvent(vertexFinishEvent);
+ // numTasks is recovered from summary log
+ assertEquals(2, vertex1.numTasks);
+ assertEquals(VertexState.SUCCEEDED, recoveredState);
+ vertex1.handle(new VertexEventRecoverVertex(vertex1.getVertexId(),
+ VertexState.SUCCEEDED));
+ dispatcher.await();
+ assertEquals(VertexState.SUCCEEDED, vertex1.getState());
+ assertEquals(vertex1.numTasks, vertex1.succeededTaskCount);
+ assertEquals(vertex1.numTasks, vertex1.completedTaskCount);
+ }
+
+ /**
* vertex1(New) -> StartRecoveryTransition(FAILED)
*/
@Test
@@ -576,7 +661,7 @@ public class TestVertexRecovery {
long finishTime = startTime + 100L;
recoveredState =
vertex1.restoreFromEvent(new VertexFinishedEvent(vertex1.getVertexId(),
- "vertex1", initRequestedTime, initedTime, startRequestedTime,
+ "vertex1", 1, initRequestedTime, initedTime, startRequestedTime,
startTime, finishTime, VertexState.SUCCEEDED, "",
new TezCounters(), new VertexStats(), null));
assertEquals(finishTime, vertex1.finishTime);
@@ -847,7 +932,7 @@ public class TestVertexRecovery {
assertEquals(VertexState.RUNNING, recoveredState);
recoveredState = vertex1.restoreFromEvent(new VertexFinishedEvent(vertex1.getVertexId(),
- "vertex1", initRequestedTime, initedTime, initRequestedTime + 300L,
+ "vertex1", 1, initRequestedTime, initedTime, initRequestedTime + 300L,
initRequestedTime + 400L, initRequestedTime + 500L,
VertexState.SUCCEEDED, "", new TezCounters(), new VertexStats(), null));
assertEquals(VertexState.SUCCEEDED, recoveredState);
http://git-wip-us.apache.org/repos/asf/tez/blob/0ebfc1b1/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java b/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java
index a7a23db..8913287 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java
@@ -361,7 +361,7 @@ public class TestHistoryEventsProtoConversion {
VertexFinishedEvent event =
new VertexFinishedEvent(TezVertexID.getInstance(
TezDAGID.getInstance(ApplicationId.newInstance(0, 1), 1), 111),
- "vertex1", 1000l, 15000l, 16000l, 20000l, 1344400l, VertexState.ERROR,
+ "vertex1", 1, 1000l, 15000l, 16000l, 20000l, 1344400l, VertexState.ERROR,
null, null, null, null);
VertexFinishedEvent deserializedEvent = (VertexFinishedEvent)
testProtoConversion(event);
@@ -376,7 +376,7 @@ public class TestHistoryEventsProtoConversion {
VertexFinishedEvent event =
new VertexFinishedEvent(TezVertexID.getInstance(
TezDAGID.getInstance(ApplicationId.newInstance(0, 1), 1), 111),
- "vertex1", 1000l, 15000l, 16000l, 20000l, 1344400l, VertexState.ERROR,
+ "vertex1", 1, 1000l, 15000l, 16000l, 20000l, 1344400l, VertexState.ERROR,
"diagnose", new TezCounters(), new VertexStats(), null);
VertexFinishedEvent deserializedEvent = (VertexFinishedEvent)
testProtoConversion(event);
http://git-wip-us.apache.org/repos/asf/tez/blob/0ebfc1b1/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java b/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java
index c3d51c3..a20c9fe 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
import org.apache.tez.common.ATSConstants;
import org.apache.tez.dag.api.EdgeManagerPluginDescriptor;
import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
@@ -139,7 +138,7 @@ public class TestHistoryEventJsonConversion {
event = new VertexParallelismUpdatedEvent(tezVertexID, 1, null, null, null, 10);
break;
case VERTEX_FINISHED:
- event = new VertexFinishedEvent(tezVertexID, "v1", random.nextInt(), random.nextInt(),
+ event = new VertexFinishedEvent(tezVertexID, "v1", 1, random.nextInt(), random.nextInt(),
random.nextInt(), random.nextInt(), random.nextInt(), VertexState.ERROR,
null, null, null, null);
break;
http://git-wip-us.apache.org/repos/asf/tez/blob/0ebfc1b1/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
index 2fed18b..a40aa74 100644
--- a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
+++ b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
@@ -140,7 +140,7 @@ public class TestHistoryEventTimelineConversion {
event = new VertexParallelismUpdatedEvent(tezVertexID, 1, null, null, null, 1);
break;
case VERTEX_FINISHED:
- event = new VertexFinishedEvent(tezVertexID, "v1", random.nextInt(), random.nextInt(),
+ event = new VertexFinishedEvent(tezVertexID, "v1", 1, random.nextInt(), random.nextInt(),
random.nextInt(), random.nextInt(), random.nextInt(), VertexState.ERROR,
null, null, null, null);
break;
@@ -448,7 +448,7 @@ public class TestHistoryEventTimelineConversion {
taskStats.put("BAR", 200);
VertexStats vertexStats = new VertexStats();
- VertexFinishedEvent event = new VertexFinishedEvent(tezVertexID, "v1", initRequestedTime,
+ VertexFinishedEvent event = new VertexFinishedEvent(tezVertexID, "v1", 1,initRequestedTime,
initedTime, startRequestedTime, startTime, finishTime, VertexState.ERROR,
"diagnostics", null, vertexStats, taskStats);
[32/50] [abbrv] tez git commit: Update CHANGES.txt post 0.5.2 release
Posted by je...@apache.org.
Update CHANGES.txt post 0.5.2 release
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/422d5d92
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/422d5d92
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/422d5d92
Branch: refs/heads/TEZ-8
Commit: 422d5d9296fe40ee1ca38aabebf3eeec108673b1
Parents: 8492f10
Author: Bikas Saha <bi...@apache.org>
Authored: Sat Nov 8 10:47:08 2014 -0800
Committer: Bikas Saha <bi...@apache.org>
Committed: Sat Nov 8 10:47:08 2014 -0800
----------------------------------------------------------------------
CHANGES.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/422d5d92/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 5749ea2..f849fec 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -19,7 +19,7 @@ ALL CHANGES:
TEZ-1747. Increase test timeout for TestSecureShuffle.
TEZ-1746. Flaky test in TestVertexImpl and TestExceptionPropagation.
-Release 0.5.2: Unreleased
+Release 0.5.2: 2014-11-07
INCOMPATIBLE CHANGES
TEZ-1666. UserPayload should be null if the payload is not specified.
[43/50] [abbrv] tez git commit: TEZ-1770. Handle ConnectExceptions
correctly when establishing connections to an NM which may be down. (sseth)
Posted by je...@apache.org.
TEZ-1770. Handle ConnectExceptions correctly when establishing
connections to an NM which may be down. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/0cceb1f2
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/0cceb1f2
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/0cceb1f2
Branch: refs/heads/TEZ-8
Commit: 0cceb1f220632e7722930315c03ca8c44c381e68
Parents: 0ebfc1b
Author: Siddharth Seth <ss...@apache.org>
Authored: Wed Nov 12 11:08:12 2014 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Wed Nov 12 11:08:12 2014 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../library/common/shuffle/HttpConnection.java | 23 ++++++++++++++++++--
2 files changed, 22 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/0cceb1f2/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 7ad6903..80263bf 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -23,6 +23,7 @@ ALL CHANGES:
TEZ-1761. TestRecoveryParser::testGetLastInProgressDAG fails in similar manner to TEZ-1686.
TEZ-1687. Use logIdentifier of Vertex for logging.
TEZ-1737. Should add taskNum in VertexFinishedEvent.
+ TEZ-1770. Handle ConnectExceptions correctly when establishing connections to an NM which may be down.
Release 0.5.2: 2014-11-07
http://git-wip-us.apache.org/repos/asf/tez/blob/0cceb1f2/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/HttpConnection.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/HttpConnection.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/HttpConnection.java
index 6e33993..4732a5a 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/HttpConnection.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/HttpConnection.java
@@ -156,12 +156,14 @@ public class HttpConnection {
connection.setConnectTimeout(unit);
int connectionFailures = 0;
while (true) {
+ long connectStartTime = System.currentTimeMillis();
try {
connection.connect();
connectionSucceeed = true;
break;
} catch (IOException ioe) {
// Don't attempt another connect if already cleanedup.
+ connectionFailures++;
if (cleanup) {
LOG.info("Cleanup is set to true. Not attempting to"
+ " connect again. Last exception was: ["
@@ -173,15 +175,32 @@ public class HttpConnection {
// throw an exception if we have waited for timeout amount of time
// note that the updated value if timeout is used here
if (connectionTimeout <= 0) {
- throw ioe;
+ throw new IOException(
+ "Failed to connect to " + url + ", #connectionFailures=" + connectionFailures, ioe);
+ }
+ long elapsed = System.currentTimeMillis() - connectStartTime;
+ if (elapsed < unit) {
+ try {
+ long sleepTime = unit - elapsed;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Sleeping for " + sleepTime + " while establishing connection to " + url +
+ ", since connectAttempt returned in " + elapsed + " ms");
+ }
+ Thread.sleep(sleepTime);
+ } catch (InterruptedException e) {
+ throw new IOException(
+ "Connection establishment sleep interrupted, #connectionFailures=" +
+ connectionFailures, e);
+ }
}
+
// reset the connect timeout for the last try
if (connectionTimeout < unit) {
unit = connectionTimeout;
// reset the connect time out for the final connect
connection.setConnectTimeout(unit);
}
- connectionFailures++;
+
}
}
if (LOG.isDebugEnabled()) {
[38/50] [abbrv] tez git commit: TEZ-1642. TestAMRecovery sometimes
fail. (zjffdu)
Posted by je...@apache.org.
TEZ-1642. TestAMRecovery sometimes fail. (zjffdu)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/6dda1d77
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/6dda1d77
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/6dda1d77
Branch: refs/heads/TEZ-8
Commit: 6dda1d77bcd6460780144ae2aab905c2a255e94a
Parents: a7d55e4
Author: Jeff Zhang <zj...@apache.org>
Authored: Tue Nov 11 09:08:04 2014 +0800
Committer: Jeff Zhang <zj...@apache.org>
Committed: Tue Nov 11 09:08:04 2014 +0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/tez/test/TestAMRecovery.java | 62 ++++++++++++++------
2 files changed, 44 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/6dda1d77/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 57a0c7f..57ca0a4 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -21,6 +21,7 @@ ALL CHANGES:
TEZ-1749. Increase test timeout for TestLocalMode.testMultipleClientsWithSession
TEZ-1750. Add a DAGScheduler which schedules tasks only when sources have been scheduled.
TEZ-1761. TestRecoveryParser::testGetLastInProgressDAG fails in similar manner to TEZ-1686.
+ TEZ-1642. TestAMRecovery sometimes fail.
Release 0.5.2: 2014-11-07
http://git-wip-us.apache.org/repos/asf/tez/blob/6dda1d77/tez-tests/src/test/java/org/apache/tez/test/TestAMRecovery.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestAMRecovery.java b/tez-tests/src/test/java/org/apache/tez/test/TestAMRecovery.java
index 42d6f5c..867c7ab 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/TestAMRecovery.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/TestAMRecovery.java
@@ -104,7 +104,7 @@ public class TestAMRecovery {
}
if (miniTezCluster == null) {
miniTezCluster =
- new MiniTezCluster(TestAMRecovery.class.getName(), 1, 1, 1);
+ new MiniTezCluster(TestAMRecovery.class.getName(), 2, 1, 1);
Configuration miniTezconf = new Configuration(conf);
miniTezconf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, MAX_AM_ATTEMPT);
miniTezconf.set("fs.defaultFS", remoteFs.getUri().toString()); // use HDFS
@@ -127,6 +127,7 @@ public class TestAMRecovery {
try {
LOG.info("Stopping MiniTezCluster");
miniTezCluster.stop();
+ miniTezCluster = null;
} catch (Exception e) {
e.printStackTrace();
}
@@ -165,6 +166,7 @@ public class TestAMRecovery {
tezConf.setBoolean(
RecoveryService.TEZ_AM_RECOVERY_HANDLE_REMAINING_EVENT_WHEN_STOPPED,
true);
+ tezConf.setInt(TezConfiguration.TEZ_AM_RM_HEARTBEAT_INTERVAL_MS_MAX, 100);
tezSession = TezClient.create("TestDAGRecovery", tezConf);
tezSession.start();
}
@@ -182,6 +184,16 @@ public class TestAMRecovery {
tezSession = null;
}
+ private void printHistoryEvents(List<HistoryEvent> historyEvents, int attemptId) {
+ LOG.info("RecoveryLogs from attempt:" + attemptId);
+ for(HistoryEvent historyEvent : historyEvents) {
+ LOG.info("Parsed event from recovery stream"
+ + ", eventType=" + historyEvent.getEventType()
+ + ", event=" + historyEvent);
+ }
+ LOG.info("");
+ }
+
/**
* Fine-grained recovery task-level, In a vertex (v1), task 0 is done task 1
* is running. History flush happens. AM dies. Once AM is recovered, task 0 is
@@ -195,14 +207,16 @@ public class TestAMRecovery {
createDAG(ControlledInputReadyVertexManager.class,
DataMovementType.BROADCAST, true);
TezCounters counters = runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
+ List<HistoryEvent> historyEvents1 = readRecoveryLog(1);
+ List<HistoryEvent> historyEvents2 = readRecoveryLog(2);
+ printHistoryEvents(historyEvents1, 1);
+ printHistoryEvents(historyEvents2, 2);
+
assertEquals(5, counters.findCounter(DAGCounter.TOTAL_LAUNCHED_TASKS).getValue());
assertEquals(1, counters.findCounter(DAGCounter.NUM_KILLED_TASKS).getValue());
assertEquals(4, counters.findCounter(DAGCounter.NUM_SUCCEEDED_TASKS).getValue());
assertEquals(2, counters.findCounter(TestCounter.Counter_1).getValue());
- List<HistoryEvent> historyEvents1 = readRecoveryLog(1);
- List<HistoryEvent> historyEvents2 = readRecoveryLog(2);
-
// task_0 of v1 is finished in attempt 1, task_1 of v1 is not finished in
// attempt 1
assertEquals(1, findTaskAttemptFinishedEvent(historyEvents1, 0, 0).size());
@@ -227,14 +241,16 @@ public class TestAMRecovery {
createDAG(ControlledInputReadyVertexManager.class,
DataMovementType.BROADCAST, false);
TezCounters counters = runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
+ List<HistoryEvent> historyEvents1 = readRecoveryLog(1);
+ List<HistoryEvent> historyEvents2 = readRecoveryLog(2);
+ printHistoryEvents(historyEvents1, 1);
+ printHistoryEvents(historyEvents2, 2);
+
assertEquals(4, counters.findCounter(DAGCounter.TOTAL_LAUNCHED_TASKS).getValue());
assertEquals(0, counters.findCounter(DAGCounter.NUM_KILLED_TASKS).getValue());
assertEquals(4, counters.findCounter(DAGCounter.NUM_SUCCEEDED_TASKS).getValue());
assertEquals(2, counters.findCounter(TestCounter.Counter_1).getValue());
- List<HistoryEvent> historyEvents1 = readRecoveryLog(1);
- List<HistoryEvent> historyEvents2 = readRecoveryLog(2);
-
// task_0 of v1 is finished in attempt 1, task_1 of v1 is not finished in
// attempt 1
assertEquals(1, findTaskAttemptFinishedEvent(historyEvents1, 0, 0).size());
@@ -259,14 +275,16 @@ public class TestAMRecovery {
createDAG(ControlledInputReadyVertexManager.class,
DataMovementType.ONE_TO_ONE, true);
TezCounters counters = runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
+ List<HistoryEvent> historyEvents1 = readRecoveryLog(1);
+ List<HistoryEvent> historyEvents2 = readRecoveryLog(2);
+ printHistoryEvents(historyEvents1, 1);
+ printHistoryEvents(historyEvents2, 2);
+
assertEquals(5, counters.findCounter(DAGCounter.TOTAL_LAUNCHED_TASKS).getValue());
assertEquals(1, counters.findCounter(DAGCounter.NUM_KILLED_TASKS).getValue());
assertEquals(4, counters.findCounter(DAGCounter.NUM_SUCCEEDED_TASKS).getValue());
assertEquals(2, counters.findCounter(TestCounter.Counter_1).getValue());
- List<HistoryEvent> historyEvents1 = readRecoveryLog(1);
- List<HistoryEvent> historyEvents2 = readRecoveryLog(2);
-
// task_0 of v1 is finished in attempt 1, task_1 of v1 is not finished in
// attempt 1
assertEquals(1, findTaskAttemptFinishedEvent(historyEvents1, 0, 0).size());
@@ -292,14 +310,16 @@ public class TestAMRecovery {
createDAG(ControlledInputReadyVertexManager.class,
DataMovementType.ONE_TO_ONE, false);
TezCounters counters = runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
+ List<HistoryEvent> historyEvents1 = readRecoveryLog(1);
+ List<HistoryEvent> historyEvents2 = readRecoveryLog(2);
+ printHistoryEvents(historyEvents1, 1);
+ printHistoryEvents(historyEvents2, 2);
+
assertEquals(4, counters.findCounter(DAGCounter.TOTAL_LAUNCHED_TASKS).getValue());
assertEquals(0, counters.findCounter(DAGCounter.NUM_KILLED_TASKS).getValue());
assertEquals(4, counters.findCounter(DAGCounter.NUM_SUCCEEDED_TASKS).getValue());
assertEquals(2, counters.findCounter(TestCounter.Counter_1).getValue());
- List<HistoryEvent> historyEvents1 = readRecoveryLog(1);
- List<HistoryEvent> historyEvents2 = readRecoveryLog(2);
-
// task_0 of v1 is finished in attempt 1, task_1 of v1 is not finished in
// attempt 1
assertEquals(1, findTaskAttemptFinishedEvent(historyEvents1, 0, 0).size());
@@ -325,14 +345,16 @@ public class TestAMRecovery {
createDAG(ControlledShuffleVertexManager.class,
DataMovementType.SCATTER_GATHER, true);
TezCounters counters = runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
+ List<HistoryEvent> historyEvents1 = readRecoveryLog(1);
+ List<HistoryEvent> historyEvents2 = readRecoveryLog(2);
+ printHistoryEvents(historyEvents1, 1);
+ printHistoryEvents(historyEvents2, 2);
+
assertEquals(5, counters.findCounter(DAGCounter.TOTAL_LAUNCHED_TASKS).getValue());
assertEquals(1, counters.findCounter(DAGCounter.NUM_KILLED_TASKS).getValue());
assertEquals(4, counters.findCounter(DAGCounter.NUM_SUCCEEDED_TASKS).getValue());
assertEquals(2, counters.findCounter(TestCounter.Counter_1).getValue());
- List<HistoryEvent> historyEvents1 = readRecoveryLog(1);
- List<HistoryEvent> historyEvents2 = readRecoveryLog(2);
-
// task_0 of v1 is finished in attempt 1, task_1 of v1 is not finished in
// attempt 1
assertEquals(1, findTaskAttemptFinishedEvent(historyEvents1, 0, 0).size());
@@ -358,14 +380,16 @@ public class TestAMRecovery {
createDAG(ControlledShuffleVertexManager.class,
DataMovementType.SCATTER_GATHER, false);
TezCounters counters = runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
+ List<HistoryEvent> historyEvents1 = readRecoveryLog(1);
+ List<HistoryEvent> historyEvents2 = readRecoveryLog(2);
+ printHistoryEvents(historyEvents1, 1);
+ printHistoryEvents(historyEvents2, 2);
+
assertEquals(4, counters.findCounter(DAGCounter.TOTAL_LAUNCHED_TASKS).getValue());
assertEquals(0, counters.findCounter(DAGCounter.NUM_KILLED_TASKS).getValue());
assertEquals(4, counters.findCounter(DAGCounter.NUM_SUCCEEDED_TASKS).getValue());
assertEquals(2, counters.findCounter(TestCounter.Counter_1).getValue());
- List<HistoryEvent> historyEvents1 = readRecoveryLog(1);
- List<HistoryEvent> historyEvents2 = readRecoveryLog(2);
-
// task_0 of v1 is finished in attempt 1, task_1 of v1 is not finished in
// attempt 1
assertEquals(1, findTaskAttemptFinishedEvent(historyEvents1, 0, 0).size());
[17/50] [abbrv] tez git commit: TEZ-1731. OnDiskMerger can end up
clobbering files across tasks with LocalDiskFetch. (sseth)
Posted by je...@apache.org.
TEZ-1731. OnDiskMerger can end up clobbering files across tasks with
LocalDiskFetch. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/57827160
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/57827160
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/57827160
Branch: refs/heads/TEZ-8
Commit: 57827160a8772858f667aacba65c6589234c641b
Parents: 8433c61
Author: Siddharth Seth <ss...@apache.org>
Authored: Mon Nov 3 13:39:49 2014 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Mon Nov 3 13:39:49 2014 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../java/org/apache/hadoop/io/FileChunk.java | 30 ++-
.../orderedgrouped/FetcherOrderedGrouped.java | 4 +-
.../shuffle/orderedgrouped/MapOutput.java | 7 +-
.../shuffle/orderedgrouped/MergeManager.java | 45 ++--
.../local/output/TezLocalTaskOutputFiles.java | 10 +-
.../common/task/local/output/TezTaskOutput.java | 7 +
.../task/local/output/TezTaskOutputFiles.java | 7 +-
.../orderedgrouped/TestMergeManager.java | 237 +++++++++++++++++++
9 files changed, 318 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/57827160/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index ce5a0d2..f741109 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -97,6 +97,7 @@ ALL CHANGES:
TEZ-1725. Fix nanosecond to millis conversion in TezMxBeanResourceCalculator.
TEZ-1726. Build broken against Hadoop-2.6.0 due to change in NodeReport.
TEZ-1579. MR examples should be setting mapreduce.framework.name to yarn-tez.
+ TEZ-1731. OnDiskMerger can end up clobbering files across tasks with LocalDiskFetch enabled.
Release 0.5.1: 2014-10-02
http://git-wip-us.apache.org/repos/asf/tez/blob/57827160/tez-runtime-library/src/main/java/org/apache/hadoop/io/FileChunk.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/hadoop/io/FileChunk.java b/tez-runtime-library/src/main/java/org/apache/hadoop/io/FileChunk.java
index a7eb90a..0ba39e2 100644
--- a/tez-runtime-library/src/main/java/org/apache/hadoop/io/FileChunk.java
+++ b/tez-runtime-library/src/main/java/org/apache/hadoop/io/FileChunk.java
@@ -18,22 +18,34 @@
package org.apache.hadoop.io;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.fs.Path;
+import org.apache.tez.runtime.library.common.InputAttemptIdentifier;
@Private
public class FileChunk implements Comparable<FileChunk> {
private final long offset;
private final long length;
- private final boolean preserveAfterUse;
+ private final boolean isLocalFile;
private final Path path;
+ private final InputAttemptIdentifier identifier;
- public FileChunk(Path path, long offset, long length, boolean preserveAfterUse) {
+ public FileChunk(Path path, long offset, long length, boolean isLocalFile,
+ InputAttemptIdentifier identifier) {
this.path = path;
this.offset = offset;
this.length = length;
- this.preserveAfterUse = preserveAfterUse;
+ this.isLocalFile = isLocalFile;
+ this.identifier = identifier;
+ if (isLocalFile) {
+ Preconditions.checkNotNull(identifier);
+ }
+ }
+
+ public FileChunk(Path path, long offset, long length) {
+ this(path, offset, length, false, null);
}
@Override
@@ -87,11 +99,15 @@ public class FileChunk implements Comparable<FileChunk> {
return length;
}
- public boolean preserveAfterUse() {
- return preserveAfterUse;
- }
-
public Path getPath() {
return path;
}
+
+ public boolean isLocalFile() {
+ return this.isLocalFile;
+ }
+
+ public InputAttemptIdentifier getInputAttemptIdentifier() {
+ return this.identifier;
+ }
}
http://git-wip-us.apache.org/repos/asf/tez/blob/57827160/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/FetcherOrderedGrouped.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/FetcherOrderedGrouped.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/FetcherOrderedGrouped.java
index 2b5a863..e83c705 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/FetcherOrderedGrouped.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/FetcherOrderedGrouped.java
@@ -441,7 +441,7 @@ class FetcherOrderedGrouped extends Thread {
LOG.debug("header: " + srcAttemptId + ", len: " + compressedLength +
", decomp len: " + decompressedLength);
}
-
+
// Get the location for the map output - either in-memory or on-disk
try {
mapOutput = merger.reserve(srcAttemptId, decompressedLength, compressedLength, id);
@@ -467,7 +467,7 @@ class FetcherOrderedGrouped extends Thread {
// Go!
LOG.info("fetcher#" + id + " about to shuffle output of map " +
mapOutput.getAttemptIdentifier() + " decomp: " +
- decompressedLength + " len: " + compressedLength);
+ decompressedLength + " len: " + compressedLength + " to " + mapOutput.getType());
if (mapOutput.getType() == Type.MEMORY) {
ShuffleUtils.shuffleToMemory(mapOutput.getMemory(), input,
(int) decompressedLength, (int) compressedLength, codec, ifileReadAhead,
http://git-wip-us.apache.org/repos/asf/tez/blob/57827160/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MapOutput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MapOutput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MapOutput.java
index c735a43..231975b 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MapOutput.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MapOutput.java
@@ -89,8 +89,11 @@ class MapOutput {
this.disk = null;
if (type == Type.DISK || type == Type.DISK_DIRECT) {
- boolean preserve = (type == Type.DISK_DIRECT); // type disk are temp files.
- this.outputPath = new FileChunk(outputPath, offset, size, preserve);
+ if (type == Type.DISK_DIRECT) {
+ this.outputPath = new FileChunk(outputPath, offset, size, true, attemptIdentifier);
+ } else {
+ this.outputPath = new FileChunk(outputPath, offset, size, false, attemptIdentifier);
+ }
} else {
this.outputPath = null;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/57827160/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java
index 0db5237..6081f91 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.Set;
import java.util.TreeSet;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -88,9 +89,11 @@ public class MergeManager {
private final Set<MapOutput> inMemoryMapOutputs =
new TreeSet<MapOutput>(new MapOutput.MapOutputComparator());
private final InMemoryMerger inMemoryMerger;
-
- private final Set<FileChunk> onDiskMapOutputs = new TreeSet<FileChunk>();
- private final OnDiskMerger onDiskMerger;
+
+ @VisibleForTesting
+ final Set<FileChunk> onDiskMapOutputs = new TreeSet<FileChunk>();
+ @VisibleForTesting
+ final OnDiskMerger onDiskMerger;
private final long memoryLimit;
private final int postMergeMemLimit;
@@ -131,7 +134,7 @@ public class MergeManager {
/**
* Construct the MergeManager. Must call start before it becomes usable.
*/
- public MergeManager(Configuration conf,
+ public MergeManager(Configuration conf,
FileSystem localFS,
LocalDirAllocator localDirAllocator,
InputContext inputContext,
@@ -213,7 +216,7 @@ public class MergeManager {
}
LOG.info("InitialRequest: ShuffleMem=" + memLimit + ", postMergeMem=" + maxRedBuffer
- + ", RuntimeTotalAvailable=" + this.initialMemoryAvailable + "Updated to: ShuffleMem="
+ + ", RuntimeTotalAvailable=" + this.initialMemoryAvailable + ". Updated to: ShuffleMem="
+ this.memoryLimit + ", postMergeMem: " + this.postMergeMemLimit);
this.ioSortFactor =
@@ -253,7 +256,7 @@ public class MergeManager {
throw new RuntimeException("Invlaid configuration: "
+ "maxSingleShuffleLimit should be less than mergeThreshold"
+ "maxSingleShuffleLimit: " + this.maxSingleShuffleLimit
- + "mergeThreshold: " + this.mergeThreshold);
+ + ", mergeThreshold: " + this.mergeThreshold);
}
boolean allowMemToMemMerge =
@@ -494,6 +497,7 @@ public class MergeManager {
return;
}
+
InputAttemptIdentifier dummyMapId = inputs.get(0).getAttemptIdentifier();
List<Segment> inMemorySegments = new ArrayList<Segment>();
long mergeOutputSize =
@@ -629,7 +633,7 @@ public class MergeManager {
}
// Note the output of the merge
- closeOnDiskFile(new FileChunk(outputPath, 0, outFileLen, false));
+ closeOnDiskFile(new FileChunk(outputPath, 0, outFileLen));
}
}
@@ -637,7 +641,8 @@ public class MergeManager {
/**
* Merges multiple on-disk segments
*/
- private class OnDiskMerger extends MergeThread<FileChunk> {
+ @VisibleForTesting
+ class OnDiskMerger extends MergeThread<FileChunk> {
public OnDiskMerger(MergeManager manager) {
super(manager, ioSortFactor, exceptionReporter);
@@ -668,7 +673,7 @@ public class MergeManager {
for (FileChunk fileChunk : inputs) {
final long offset = fileChunk.getOffset();
final long size = fileChunk.getLength();
- final boolean preserve = fileChunk.preserveAfterUse();
+ final boolean preserve = fileChunk.isLocalFile();
final Path file = fileChunk.getPath();
approxOutputSize += size;
Segment segment = new Segment(conf, rfs, file, offset, size, codec, ifileReadAhead,
@@ -681,9 +686,19 @@ public class MergeManager {
ChecksumFileSystem.getChecksumLength(approxOutputSize, bytesPerSum);
// 2. Start the on-disk merge process
- Path outputPath =
- localDirAllocator.getLocalPathForWrite(inputs.get(0).getPath().toString(),
- approxOutputSize, conf).suffix(Constants.MERGED_OUTPUT_PREFIX);
+ FileChunk file0 = inputs.get(0);
+ String namePart;
+ if (file0.isLocalFile()) {
+ // This is setup the same way a type DISK MapOutput is setup when fetching.
+ namePart = mapOutputFile.getSpillFileName(
+ file0.getInputAttemptIdentifier().getInputIdentifier().getInputIndex());
+
+ } else {
+ namePart = file0.getPath().getName().toString();
+ }
+ Path outputPath = localDirAllocator.getLocalPathForWrite(namePart, approxOutputSize, conf);
+ outputPath = outputPath.suffix(Constants.MERGED_OUTPUT_PREFIX);
+
Writer writer =
new Writer(conf, rfs, outputPath,
(Class)ConfigUtils.getIntermediateInputKeyClass(conf),
@@ -712,7 +727,7 @@ public class MergeManager {
}
final long outputLen = localFS.getFileStatus(outputPath).getLen();
- closeOnDiskFile(new FileChunk(outputPath, 0, outputLen, false));
+ closeOnDiskFile(new FileChunk(outputPath, 0, outputLen));
LOG.info(inputContext.getUniqueIdentifier() +
" Finished merging " + inputs.size() +
@@ -857,7 +872,7 @@ public class MergeManager {
final FileStatus fStatus = localFS.getFileStatus(outputPath);
// add to list of final disk outputs.
- onDiskMapOutputs.add(new FileChunk(outputPath, 0, fStatus.getLen(), false));
+ onDiskMapOutputs.add(new FileChunk(outputPath, 0, fStatus.getLen()));
LOG.info("Merged " + numMemDiskSegments + " segments, " +
inMemToDiskBytes + " bytes to disk to satisfy " +
@@ -885,7 +900,7 @@ public class MergeManager {
file.toString().endsWith(Constants.MERGED_OUTPUT_PREFIX) ? null : mergedMapOutputsCounter;
final long fileOffset = fileChunk.getOffset();
- final boolean preserve = fileChunk.preserveAfterUse();
+ final boolean preserve = fileChunk.isLocalFile();
diskSegments.add(new Segment(job, fs, file, fileOffset, fileLength, codec, ifileReadAhead,
ifileReadAheadLength, ifileBufferSize, preserve, counter));
}
http://git-wip-us.apache.org/repos/asf/tez/blob/57827160/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/TezLocalTaskOutputFiles.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/TezLocalTaskOutputFiles.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/TezLocalTaskOutputFiles.java
index 3d83010..e02011f 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/TezLocalTaskOutputFiles.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/TezLocalTaskOutputFiles.java
@@ -222,11 +222,17 @@ public class TezLocalTaskOutputFiles extends TezTaskOutput {
public Path getInputFileForWrite(int taskId,
long size)
throws IOException {
- return lDirAlloc.getLocalPathForWrite(String.format(
- Constants.TEZ_RUNTIME_TASK_INPUT_FILE_FORMAT_STRING, Constants.TEZ_RUNTIME_TASK_OUTPUT_DIR, taskId),
+ return lDirAlloc.getLocalPathForWrite(getSpillFileName(taskId),
size, conf);
}
+ @Override
+ public String getSpillFileName(int spillNum) {
+ return (String.format(
+ Constants.TEZ_RUNTIME_TASK_INPUT_FILE_FORMAT_STRING, Constants.TEZ_RUNTIME_TASK_OUTPUT_DIR,
+ spillNum));
+ }
+
/** Removes all of the files related to a task. */
@Override
public void removeAll()
http://git-wip-us.apache.org/repos/asf/tez/blob/57827160/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/TezTaskOutput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/TezTaskOutput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/TezTaskOutput.java
index d3e7d27..e34e399 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/TezTaskOutput.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/TezTaskOutput.java
@@ -160,6 +160,13 @@ public abstract class TezTaskOutput {
public abstract Path getInputFileForWrite(
int taskIdentifier, long size) throws IOException;
+ /**
+ * Construct a spill file name, given a spill number
+ * @param spillNum
+ * @return
+ */
+ public abstract String getSpillFileName(int spillNum);
+
/** Removes all of the files related to a task. */
public abstract void removeAll() throws IOException;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/57827160/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/TezTaskOutputFiles.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/TezTaskOutputFiles.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/TezTaskOutputFiles.java
index 8ae1a04..86a83ac 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/TezTaskOutputFiles.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/TezTaskOutputFiles.java
@@ -234,11 +234,14 @@ public class TezTaskOutputFiles extends TezTaskOutput {
*/
public Path getInputFileForWrite(int srcTaskId,
long size) throws IOException {
- return lDirAlloc.getLocalPathForWrite(String.format(SPILL_FILE_PATTERN,
- uniqueId, srcTaskId),
+ return lDirAlloc.getLocalPathForWrite(getSpillFileName(srcTaskId),
size, conf);
}
+ public String getSpillFileName(int spillNum) {
+ return String.format(SPILL_FILE_PATTERN, uniqueId, spillNum);
+ }
+
/** Removes all of the files related to a task. */
public void removeAll() throws IOException {
throw new UnsupportedOperationException("Incompatible with LocalRunner");
http://git-wip-us.apache.org/repos/asf/tez/blob/57827160/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestMergeManager.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestMergeManager.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestMergeManager.java
new file mode 100644
index 0000000..7615ba7
--- /dev/null
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestMergeManager.java
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.runtime.library.common.shuffle.orderedgrouped;
+
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.UUID;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalDirAllocator;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.FileChunk;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.tez.common.TezRuntimeFrameworkConfigs;
+import org.apache.tez.common.counters.TezCounters;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.runtime.api.InputContext;
+import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
+import org.apache.tez.runtime.library.common.InputAttemptIdentifier;
+import org.apache.tez.runtime.library.common.sort.impl.IFile;
+import org.apache.tez.runtime.library.common.sort.impl.TezIndexRecord;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestMergeManager {
+
+
+ private static final Log LOG = LogFactory.getLog(TestMergeManager.class);
+
+ private static Configuration defaultConf = new TezConfiguration();
+ private static FileSystem localFs = null;
+ private static Path workDir = null;
+
+ static {
+ try {
+ defaultConf.set("fs.defaultFS", "file:///");
+ localFs = FileSystem.getLocal(defaultConf);
+ workDir =
+ new Path(new Path(System.getProperty("test.build.data", "/tmp")),
+ TestMergeManager.class.getSimpleName());
+ workDir = localFs.makeQualified(workDir);
+ localFs.mkdirs(workDir);
+ LOG.info("Using workDir: " + workDir);
+ } catch (IOException e) {
+ throw new RuntimeException("init failure", e);
+ }
+ }
+
+ @Before
+ @After
+ public void cleanup() throws IOException {
+ localFs.delete(workDir, true);
+ }
+
+ @Test(timeout = 10000)
+ public void testLocalDiskMergeMultipleTasks() throws IOException {
+
+ Configuration conf = new TezConfiguration(defaultConf);
+ conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, false);
+ conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, IntWritable.class.getName());
+ conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, IntWritable.class.getName());
+
+ Path localDir = new Path(workDir, "local");
+ Path srcDir = new Path(workDir, "srcData");
+ localFs.mkdirs(localDir);
+ localFs.mkdirs(srcDir);
+
+ conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, localDir.toString());
+
+ FileSystem localFs = FileSystem.getLocal(conf);
+ LocalDirAllocator localDirAllocator =
+ new LocalDirAllocator(TezRuntimeFrameworkConfigs.LOCAL_DIRS);
+ InputContext t0inputContext = createMockInputContext(UUID.randomUUID().toString());
+ InputContext t1inputContext = createMockInputContext(UUID.randomUUID().toString());
+
+ ExceptionReporter t0exceptionReporter = mock(ExceptionReporter.class);
+ ExceptionReporter t1exceptionReporter = mock(ExceptionReporter.class);
+
+ MergeManager t0mergeManagerReal =
+ new MergeManager(conf, localFs, localDirAllocator, t0inputContext, null, null, null, null,
+ t0exceptionReporter, 2000000, null, false, -1);
+ MergeManager t0mergeManager = spy(t0mergeManagerReal);
+
+ MergeManager t1mergeManagerReal =
+ new MergeManager(conf, localFs, localDirAllocator, t1inputContext, null, null, null, null,
+ t1exceptionReporter, 2000000, null, false, -1);
+ MergeManager t1mergeManager = spy(t1mergeManagerReal);
+
+ // Partition 0 Keys 0-2, Partition 1 Keys 3-5
+ SrcFileInfo src1Info =
+ createFile(conf, localFs, new Path(srcDir, InputAttemptIdentifier.PATH_PREFIX + "src1.out"),
+ 2, 3, 0);
+ // Partition 0 Keys 6-8, Partition 1 Keys 9-11
+ SrcFileInfo src2Info =
+ createFile(conf, localFs, new Path(srcDir, InputAttemptIdentifier.PATH_PREFIX + "src2.out"),
+ 2, 3, 6);
+
+
+ // Simulating Task 0 fetches partition 0. (targetIndex = 0,1)
+
+ // Simulating Task 1 fetches partition 1. (targetIndex = 0,1)
+
+ InputAttemptIdentifier t0Identifier0 =
+ new InputAttemptIdentifier(0, 0, src1Info.path.getName());
+ InputAttemptIdentifier t0Identifier1 =
+ new InputAttemptIdentifier(1, 0, src2Info.path.getName());
+
+ InputAttemptIdentifier t1Identifier0 =
+ new InputAttemptIdentifier(0, 0, src1Info.path.getName());
+ InputAttemptIdentifier t1Identifier1 =
+ new InputAttemptIdentifier(1, 0, src2Info.path.getName());
+
+
+ MapOutput t0MapOutput0 =
+ getMapOutputForDirectDiskFetch(t0Identifier0, src1Info.path, src1Info.indexedRecords[0],
+ t0mergeManager);
+ MapOutput t0MapOutput1 =
+ getMapOutputForDirectDiskFetch(t0Identifier1, src2Info.path, src2Info.indexedRecords[0],
+ t0mergeManager);
+
+ MapOutput t1MapOutput0 =
+ getMapOutputForDirectDiskFetch(t1Identifier0, src1Info.path, src1Info.indexedRecords[1],
+ t1mergeManager);
+ MapOutput t1MapOutput1 =
+ getMapOutputForDirectDiskFetch(t1Identifier1, src2Info.path, src2Info.indexedRecords[1],
+ t1mergeManager);
+
+
+ t0MapOutput0.commit();
+ t0MapOutput1.commit();
+ verify(t0mergeManager).closeOnDiskFile(t0MapOutput0.getOutputPath());
+ verify(t0mergeManager).closeOnDiskFile(t0MapOutput1.getOutputPath());
+ // Run the OnDiskMerge via MergeManager
+ // Simulate the thread invocation - remove files, and invoke merge
+ List<FileChunk> t0MergeFiles = new LinkedList<FileChunk>();
+ t0MergeFiles.addAll(t0mergeManager.onDiskMapOutputs);
+ t0mergeManager.onDiskMapOutputs.clear();
+ t0mergeManager.onDiskMerger.merge(t0MergeFiles);
+ Assert.assertEquals(1, t0mergeManager.onDiskMapOutputs.size());
+
+
+ t1MapOutput0.commit();
+ t1MapOutput1.commit();
+ verify(t1mergeManager).closeOnDiskFile(t1MapOutput0.getOutputPath());
+ verify(t1mergeManager).closeOnDiskFile(t1MapOutput1.getOutputPath());
+ // Run the OnDiskMerge via MergeManager
+ // Simulate the thread invocation - remove files, and invoke merge
+ List<FileChunk> t1MergeFiles = new LinkedList<FileChunk>();
+ t1MergeFiles.addAll(t1mergeManager.onDiskMapOutputs);
+ t1mergeManager.onDiskMapOutputs.clear();
+ t1mergeManager.onDiskMerger.merge(t1MergeFiles);
+ Assert.assertEquals(1, t1mergeManager.onDiskMapOutputs.size());
+
+ Assert.assertNotEquals(t0mergeManager.onDiskMapOutputs.iterator().next().getPath(),
+ t1mergeManager.onDiskMapOutputs.iterator().next().getPath());
+
+ Assert.assertTrue(t0mergeManager.onDiskMapOutputs.iterator().next().getPath().toString()
+ .contains(t0inputContext.getUniqueIdentifier()));
+ Assert.assertTrue(t1mergeManager.onDiskMapOutputs.iterator().next().getPath().toString()
+ .contains(t1inputContext.getUniqueIdentifier()));
+
+ }
+
+ private InputContext createMockInputContext(String uniqueId) {
+ InputContext inputContext = mock(InputContext.class);
+ doReturn(new TezCounters()).when(inputContext).getCounters();
+ doReturn(200 * 1024 * 1024l).when(inputContext).getTotalMemoryAvailableToTask();
+ doReturn("srcVertexName").when(inputContext).getSourceVertexName();
+ doReturn(uniqueId).when(inputContext).getUniqueIdentifier();
+ return inputContext;
+ }
+
+ private SrcFileInfo createFile(Configuration conf, FileSystem fs, Path path, int numPartitions,
+ int numKeysPerPartition, int startKey) throws IOException {
+ FSDataOutputStream outStream = fs.create(path);
+ int currentKey = startKey;
+ SrcFileInfo srcFileInfo = new SrcFileInfo();
+ srcFileInfo.indexedRecords = new TezIndexRecord[numPartitions];
+ srcFileInfo.path = path;
+ for (int i = 0; i < numPartitions; i++) {
+ long pos = outStream.getPos();
+ IFile.Writer writer =
+ new IFile.Writer(conf, outStream, IntWritable.class, IntWritable.class, null, null, null);
+ for (int j = 0; j < numKeysPerPartition; j++) {
+ writer.append(new IntWritable(currentKey), new IntWritable(currentKey));
+ currentKey++;
+ }
+ writer.close();
+ srcFileInfo.indexedRecords[i] =
+ new TezIndexRecord(pos, writer.getRawLength(), writer.getCompressedLength());
+ }
+ outStream.close();
+ return srcFileInfo;
+ }
+
+ private class SrcFileInfo {
+ private Path path;
+ private TezIndexRecord[] indexedRecords;
+ }
+
+ // Copied from FetcherOrderedGrouped
+ private static MapOutput getMapOutputForDirectDiskFetch(InputAttemptIdentifier srcAttemptId,
+ Path filename, TezIndexRecord indexRecord,
+ MergeManager merger)
+ throws IOException {
+ return MapOutput.createLocalDiskMapOutput(srcAttemptId, merger, filename,
+ indexRecord.getStartOffset(), indexRecord.getPartLength(), true);
+ }
+}
[28/50] [abbrv] tez git commit: TEZ-1746. Flaky test in
TestVertexImpl and TestExceptionPropagation. (zjffdu)
Posted by je...@apache.org.
TEZ-1746. Flaky test in TestVertexImpl and TestExceptionPropagation. (zjffdu)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/873b247b
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/873b247b
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/873b247b
Branch: refs/heads/TEZ-8
Commit: 873b247bbdc930572d08b29aef84d8cb65d4c16d
Parents: 7c6085d
Author: Jeff Zhang <zj...@apache.org>
Authored: Fri Nov 7 11:48:59 2014 +0800
Committer: Jeff Zhang <zj...@apache.org>
Committed: Fri Nov 7 11:48:59 2014 +0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../tez/dag/app/dag/impl/TestVertexImpl.java | 32 ++++++++++----------
.../tez/test/TestExceptionPropagation.java | 2 +-
3 files changed, 18 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/873b247b/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index e5a4eed..359f43c 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -15,6 +15,7 @@ Release 0.5.3: Unreleased
ALL CHANGES:
TEZ-1745. TestATSHistoryLoggingService::testATSHistoryLoggingServiceShutdown can be flaky.
TEZ-1747. Increase test timeout for TestSecureShuffle.
+ TEZ-1746. Flaky test in TestVertexImpl and TestExceptionPropagation.
Release 0.5.2: Unreleased
http://git-wip-us.apache.org/repos/asf/tez/blob/873b247b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
index 97c0693..9500c97 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
@@ -513,19 +513,6 @@ public class TestVertexImpl {
.setTaskModule("x2.y2")
.build()
)
- .addInputs(
- RootInputLeafOutputProto.newBuilder()
- .setControllerDescriptor(
- TezEntityDescriptorProto.newBuilder().setClassName(
- "IrrelevantInitializerClassName"))
- .setName("input1")
- .setIODescriptor(
- TezEntityDescriptorProto.newBuilder()
- .setClassName("InputClazz")
- .build()
- )
- .build()
- )
.addInEdgeId("e1")
.build()
)
@@ -5072,7 +5059,7 @@ public class TestVertexImpl {
}
@Test(timeout = 5000)
- public void testExceptionFromII_Initialize() throws AMUserCodeException {
+ public void testExceptionFromII_Initialize() throws AMUserCodeException, InterruptedException {
useCustomInitializer = true;
customInitializer = new EventHandlingRootInputInitializer(null, IIExceptionLocation.Initialize);
EventHandlingRootInputInitializer initializer =
@@ -5084,6 +5071,11 @@ public class TestVertexImpl {
VertexImplWithRunningInputInitializer v1 =
(VertexImplWithRunningInputInitializer) vertices.get("vertex1");
initVertex(v1);
+ // Wait for the initializer to been invoked, and fail the vertex finally.
+ while (v1.getState() != VertexState.FAILED) {
+ Thread.sleep(10);
+ }
+
String diagnostics = StringUtils.join(v1.getDiagnostics(), ",");
assertTrue(diagnostics.contains(IIExceptionLocation.Initialize.name()));
Assert.assertEquals(VertexState.FAILED, v1.getState());
@@ -5206,6 +5198,10 @@ public class TestVertexImpl {
(VertexImplWithRunningInputInitializer) vertices.get("vertex2");
initVertex(v1);
+ // Wait for the initializer to be invoked - which may be a separate thread.
+ while (!initializer.initStarted.get()) {
+ Thread.sleep(10);
+ }
startVertex(v1); // v2 would get the state update from v1
dispatcher.await();
Assert.assertEquals(VertexState.RUNNING, v1.getState());
@@ -5232,6 +5228,10 @@ public class TestVertexImpl {
(VertexImplWithRunningInputInitializer) vertices.get("vertex2");
initVertex(v1);
+ // Wait for the initializer to be invoked - which may be a separate thread.
+ while (!initializer.initStarted.get()) {
+ Thread.sleep(10);
+ }
startVertex(v1); // v2 would get the state update from v1
// it should be OK receive INIT_SUCCEEDED event after INIT_FAILED event
dispatcher.getEventHandler().handle(new VertexEventRootInputInitialized(
@@ -5441,11 +5441,11 @@ public class TestVertexImpl {
@Override
public List<Event> initialize() throws Exception {
+ context.registerForVertexStateUpdates("vertex1", null);
+ initStarted.set(true);
if (exLocation == IIExceptionLocation.Initialize) {
throw new Exception(exLocation.name());
}
- context.registerForVertexStateUpdates("vertex1", null);
- initStarted.set(true);
lock.lock();
try {
if (!eventReceived.get()) {
http://git-wip-us.apache.org/repos/asf/tez/blob/873b247b/tez-tests/src/test/java/org/apache/tez/test/TestExceptionPropagation.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestExceptionPropagation.java b/tez-tests/src/test/java/org/apache/tez/test/TestExceptionPropagation.java
index 0175d7b..efb6eb6 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/TestExceptionPropagation.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/TestExceptionPropagation.java
@@ -212,7 +212,7 @@ public class TestExceptionPropagation {
* @throws Exception
*
*/
- @Test(timeout = 180000)
+ @Test(timeout = 300000)
public void testExceptionPropagationSession() throws Exception {
try {
startSessionClient();
[16/50] [abbrv] tez git commit: Move TEZ-1579 to the 0.5.2 section in
CHANGES.txt
Posted by je...@apache.org.
Move TEZ-1579 to the 0.5.2 section in CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/8433c611
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/8433c611
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/8433c611
Branch: refs/heads/TEZ-8
Commit: 8433c611932e0e82d6826491d4f1b915cadafc60
Parents: b768c0f
Author: Siddharth Seth <ss...@apache.org>
Authored: Sat Nov 1 12:42:34 2014 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Sat Nov 1 12:42:34 2014 -0700
----------------------------------------------------------------------
CHANGES.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/8433c611/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index d43ccf7..ce5a0d2 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -6,7 +6,6 @@ Release 0.6.0: Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES:
- TEZ-1579. MR examples should be setting mapreduce.framework.name to yarn-tez.
Release 0.5.2: Unreleased
@@ -97,6 +96,7 @@ ALL CHANGES:
TEZ-1703. addendum - fix flaky test.
TEZ-1725. Fix nanosecond to millis conversion in TezMxBeanResourceCalculator.
TEZ-1726. Build broken against Hadoop-2.6.0 due to change in NodeReport.
+ TEZ-1579. MR examples should be setting mapreduce.framework.name to yarn-tez.
Release 0.5.1: 2014-10-02
[44/50] [abbrv] tez git commit: TEZ-1736. Add support for
Inputs/Outputs in runtime-library to generate history text data. (hitesh)
Posted by je...@apache.org.
TEZ-1736. Add support for Inputs/Outputs in runtime-library to generate history text data. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/0daf2ba1
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/0daf2ba1
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/0daf2ba1
Branch: refs/heads/TEZ-8
Commit: 0daf2ba188416f69198e23621d05f3dd3acd2882
Parents: 0cceb1f
Author: Hitesh Shah <hi...@apache.org>
Authored: Wed Nov 12 14:28:10 2014 -0800
Committer: Hitesh Shah <hi...@apache.org>
Committed: Wed Nov 12 14:28:10 2014 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/tez/common/ATSConstants.java | 6 ++-
.../java/org/apache/tez/common/TezUtils.java | 36 +++++++++++++
.../apache/tez/dag/api/EntityDescriptor.java | 6 ++-
.../org/apache/tez/common/TestTezUtils.java | 57 +++++++++++++++++---
.../apache/tez/mapreduce/client/YARNRunner.java | 14 +++++
.../org/apache/tez/mapreduce/input/MRInput.java | 26 ++++++++-
.../apache/tez/mapreduce/output/MROutput.java | 6 +++
.../logging/ats/ATSHistoryLoggingService.java | 8 +--
.../library/api/TezRuntimeConfiguration.java | 8 +++
.../HadoopKeyValuesBasedBaseEdgeConfig.java | 13 +++++
.../conf/OrderedGroupedKVInputConfig.java | 9 ++++
.../conf/OrderedPartitionedKVEdgeConfig.java | 12 +++++
.../conf/OrderedPartitionedKVOutputConfig.java | 9 ++++
.../library/conf/UnorderedKVEdgeConfig.java | 13 +++++
.../library/conf/UnorderedKVInputConfig.java | 9 ++++
.../library/conf/UnorderedKVOutputConfig.java | 9 ++++
.../conf/UnorderedPartitionedKVEdgeConfig.java | 12 +++++
.../UnorderedPartitionedKVOutputConfig.java | 8 +++
.../apache/tez/runtime/library/conf/Utils.java | 45 ++++++++++++++++
.../library/input/OrderedGroupedKVInput.java | 1 +
.../runtime/library/input/UnorderedKVInput.java | 1 +
.../output/OrderedPartitionedKVOutput.java | 1 +
.../library/output/UnorderedKVOutput.java | 1 +
.../output/UnorderedPartitionedKVOutput.java | 1 +
.../TestOrderedPartitionedKVEdgeConfig.java | 45 ++++++++++++++--
.../library/conf/TestUnorderedKVEdgeConfig.java | 48 +++++++++++++++--
.../TestUnorderedPartitionedKVEdgeConfig.java | 44 +++++++++++++--
.../apache/tez/mapreduce/TestMRRJobsDAGApi.java | 10 +++-
29 files changed, 430 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/0daf2ba1/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 80263bf..aa7d74d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,7 @@ ALL CHANGES:
TEZ-1733. TezMerger should sort FileChunks on size when merging
TEZ-1738. Tez tfile parser for log parsing
TEZ-1627. Remove OUTPUT_CONSUMABLE and related Event in TaskAttemptImpl
+ TEZ-1736. Add support for Inputs/Outputs in runtime-library to generate history text data.
Release 0.5.3: Unreleased
http://git-wip-us.apache.org/repos/asf/tez/blob/0daf2ba1/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java b/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java
index 58761d5..7b47b9c 100644
--- a/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java
+++ b/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java
@@ -50,7 +50,6 @@ public class ATSConstants {
public static final String APP_SUBMIT_TIME = "appSubmitTime";
/* Tez-specific info */
- public static final String CONFIG = "config";
public static final String DAG_PLAN = "dagPlan";
public static final String DAG_NAME = "dagName";
public static final String VERTEX_NAME = "vertexName";
@@ -99,4 +98,9 @@ public class ATSConstants {
"yarn.timeline-service.webapp.address";
public static final String TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS_CONF_NAME =
"yarn.timeline-service.webapp.https.address";
+
+ /* History text related Keys */
+ public static final String DESCRIPTION = "desc";
+ public static final String CONFIG = "config";
+
}
http://git-wip-us.apache.org/repos/asf/tez/blob/0daf2ba1/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
index c008f35..fb2ed78 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
@@ -23,17 +23,24 @@ import java.io.OutputStream;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.zip.Deflater;
import java.util.zip.DeflaterOutputStream;
import java.util.zip.InflaterInputStream;
import com.google.common.base.Preconditions;
import com.google.protobuf.ByteString;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.tez.client.TezClientUtils;
+import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.api.UserPayload;
import org.apache.tez.dag.api.records.DAGProtos;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
/**
@@ -43,6 +50,7 @@ import org.apache.tez.dag.api.records.DAGProtos;
@InterfaceAudience.Public
public class TezUtils {
+ private static final Log LOG = LogFactory.getLog(TezUtils.class);
/**
* Allows changing the log level for task / AM logging. </p>
@@ -148,4 +156,32 @@ public class TezUtils {
conf.set(setting.getKey(), setting.getValue());
}
}
+
+ public static String convertToHistoryText(String description, Configuration conf) {
+ // Add a version if this serialization is changed
+ JSONObject jsonObject = new JSONObject();
+ try {
+ if (description != null && !description.isEmpty()) {
+ jsonObject.put(ATSConstants.DESCRIPTION, description);
+ }
+ if (conf != null) {
+ JSONObject confJson = new JSONObject();
+ Iterator<Entry<String, String>> iter = conf.iterator();
+ while (iter.hasNext()) {
+ Entry<String, String> entry = iter.next();
+ confJson.put(entry.getKey(), entry.getValue());
+ }
+ jsonObject.put(ATSConstants.CONFIG, confJson);
+ }
+ } catch (JSONException e) {
+ throw new TezUncheckedException("Error when trying to convert description/conf to JSON", e);
+ }
+ return jsonObject.toString();
+ }
+
+ public static String convertToHistoryText(Configuration conf) {
+ return convertToHistoryText(null, conf);
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/tez/blob/0daf2ba1/tez-api/src/main/java/org/apache/tez/dag/api/EntityDescriptor.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/EntityDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/EntityDescriptor.java
index 2caf819..d02bddd 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/EntityDescriptor.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/EntityDescriptor.java
@@ -70,8 +70,12 @@ public abstract class EntityDescriptor<T extends EntityDescriptor<T>> implements
/**
* Provide a human-readable version of the user payload that can be
- * used in the History UI
+ * used in the TEZ UI
* @param historyText History text
+ * For better support in the UI, the history text should be a json-encoded string.
+ * The following keys in the json object will be recognized:
+ * "desc" : A string-value describing the entity
+ * "config" : A key-value map to represent configuration
* @return this object for further chained method calls
*/
public T setHistoryText(String historyText) {
http://git-wip-us.apache.org/repos/asf/tez/blob/0daf2ba1/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java
----------------------------------------------------------------------
diff --git a/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java b/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java
index e1036a5..99408f8 100644
--- a/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java
+++ b/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java
@@ -23,13 +23,16 @@ import java.util.Random;
import org.apache.hadoop.conf.Configuration;
import org.apache.tez.dag.api.UserPayload;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
import org.junit.Assert;
import org.junit.Test;
import com.google.protobuf.ByteString;
public class TestTezUtils {
- @Test
+
+ @Test (timeout=2000)
public void testByteStringToAndFromConf() throws IOException {
Configuration conf = getConf();
Assert.assertEquals(conf.size(), 6);
@@ -41,7 +44,7 @@ public class TestTezUtils {
checkConf(conf);
}
- @Test
+ @Test (timeout=2000)
public void testPayloadToAndFromConf() throws IOException {
Configuration conf = getConf();
Assert.assertEquals(conf.size(), 6);
@@ -52,8 +55,8 @@ public class TestTezUtils {
Assert.assertEquals(conf.size(), 6);
checkConf(conf);
}
-
- @Test
+
+ @Test (timeout=2000)
public void testCleanVertexName() {
String testString = "special characters & spaces and longer than "
+ TezUtilsInternal.MAX_VERTEX_NAME_LENGTH + " characters";
@@ -64,7 +67,7 @@ public class TestTezUtils {
Assert.assertTrue(cleaned.matches("\\w+"));
}
- @Test
+ @Test (timeout=2000)
public void testBitSetToByteArray() {
BitSet bitSet = createBitSet(0);
byte[] bytes = TezUtilsInternal.toByteArray(bitSet);
@@ -75,7 +78,7 @@ public class TestTezUtils {
Assert.assertTrue(bytes.length == ((bitSet.length() / 8) + 1));
}
- @Test
+ @Test (timeout=2000)
public void testBitSetFromByteArray() {
BitSet bitSet = createBitSet(0);
byte[] bytes = TezUtilsInternal.toByteArray(bitSet);
@@ -93,7 +96,7 @@ public class TestTezUtils {
Assert.assertTrue(TezUtilsInternal.fromByteArray(bytes).equals(bitSet));
}
- @Test
+ @Test (timeout=2000)
public void testBitSetConversion() {
for (int i = 0 ; i < 16 ; i++) {
BitSet bitSet = createBitSetWithSingleEntry(i);
@@ -146,4 +149,44 @@ public class TestTezUtils {
Assert.assertEquals(tmp[2], "S3");
}
+
+ @Test (timeout=2000)
+ public void testConvertToHistoryText() throws JSONException {
+ Configuration conf = getConf();
+
+ String confToJson = TezUtils.convertToHistoryText(conf);
+
+ JSONObject jsonObject = new JSONObject(confToJson);
+
+ Assert.assertFalse(jsonObject.has(ATSConstants.DESCRIPTION));
+ Assert.assertTrue(jsonObject.has(ATSConstants.CONFIG));
+
+ JSONObject confObject = jsonObject.getJSONObject(ATSConstants.CONFIG);
+ Assert.assertNotNull(confObject);
+ Assert.assertEquals("value1", confObject.getString("test1"));
+ Assert.assertEquals("true", confObject.getString("test2"));
+ Assert.assertEquals("1.2345", confObject.getString("test3"));
+ Assert.assertEquals("34567", confObject.getString("test4"));
+ Assert.assertEquals("1234567890", confObject.getString("test5"));
+ Assert.assertEquals("S1,S2,S3", confObject.getString("test6"));
+
+ String desc = "desc123";
+ confToJson = TezUtils.convertToHistoryText(desc, conf);
+ jsonObject = new JSONObject(confToJson);
+
+ Assert.assertTrue(jsonObject.has(ATSConstants.DESCRIPTION));
+ String descFromJson = jsonObject.getString(ATSConstants.DESCRIPTION);
+ Assert.assertEquals(desc, descFromJson);
+
+ Assert.assertTrue(jsonObject.has(ATSConstants.CONFIG));
+ confObject = jsonObject.getJSONObject("config");
+ Assert.assertNotNull(confObject);
+ Assert.assertEquals("value1", confObject.getString("test1"));
+ Assert.assertEquals("true", confObject.getString("test2"));
+ Assert.assertEquals("1.2345", confObject.getString("test3"));
+ Assert.assertEquals("34567", confObject.getString("test4"));
+ Assert.assertEquals("1234567890", confObject.getString("test5"));
+ Assert.assertEquals("S1,S2,S3", confObject.getString("test6"));
+
+ }
}
http://git-wip-us.apache.org/repos/asf/tez/blob/0daf2ba1/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
index dfbf0cf..1cba105 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
@@ -420,6 +420,11 @@ public class YARNRunner implements ClientProtocol {
Vertex vertex = Vertex.create(vertexName,
ProcessorDescriptor.create(processorName).setUserPayload(vertexUserPayload),
numTasks, taskResource);
+ if (stageConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT,
+ TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT_DEFAULT)) {
+ vertex.getProcessorDescriptor().setHistoryText(TezUtils.convertToHistoryText(stageConf));
+ }
+
if (isMap) {
vertex.addDataSource("MRInput",
configureMRInputWithLegacySplitsGenerated(stageConf, true));
@@ -428,6 +433,10 @@ public class YARNRunner implements ClientProtocol {
if (stageNum == totalStages -1) {
OutputDescriptor od = OutputDescriptor.create(MROutputLegacy.class.getName())
.setUserPayload(vertexUserPayload);
+ if (stageConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT,
+ TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT_DEFAULT)) {
+ od.setHistoryText(TezUtils.convertToHistoryText(stageConf));
+ }
vertex.addDataSink("MROutput", DataSinkDescriptor.create(od,
OutputCommitterDescriptor.create(MROutputCommitter.class.getName()), null));
}
@@ -806,6 +815,11 @@ public class YARNRunner implements ClientProtocol {
}
DataSourceDescriptor dsd = DataSourceDescriptor.create(inputDescriptor, null, null);
+ if (conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT,
+ TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT_DEFAULT)) {
+ dsd.getInputDescriptor().setHistoryText(TezUtils.convertToHistoryText(conf));
+ }
+
return dsd;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/0daf2ba1/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java
index f38fc9c..5c89f0e 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitIndex;
import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitMetaInfo;
import org.apache.hadoop.security.Credentials;
+import org.apache.tez.common.TezUtils;
import org.apache.tez.common.counters.TaskCounter;
import org.apache.tez.dag.api.DataSourceDescriptor;
import org.apache.tez.dag.api.InputDescriptor;
@@ -65,6 +66,7 @@ import org.apache.tez.runtime.api.Input;
import org.apache.tez.runtime.api.InputContext;
import org.apache.tez.runtime.api.events.InputDataInformationEvent;
import org.apache.tez.runtime.library.api.KeyValueReader;
+import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
@@ -250,16 +252,23 @@ public class MRInput extends MRInputBase {
}
MRHelpers.translateMRConfToTez(conf);
- UserPayload payload = MRInputHelpersInternal.createMRInputPayload(conf, inputSplitInfo.getSplitsProto());
+ UserPayload payload = MRInputHelpersInternal.createMRInputPayload(conf,
+ inputSplitInfo.getSplitsProto());
Credentials credentials = null;
if (getCredentialsForSourceFilesystem && inputSplitInfo.getCredentials() != null) {
credentials = inputSplitInfo.getCredentials();
}
- return DataSourceDescriptor.create(
+ DataSourceDescriptor ds = DataSourceDescriptor.create(
InputDescriptor.create(inputClassName).setUserPayload(payload),
InputInitializerDescriptor.create(MRInputSplitDistributor.class.getName()),
inputSplitInfo.getNumTasks(), credentials,
VertexLocationHint.create(inputSplitInfo.getTaskLocationHints()), null);
+ if (conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT,
+ TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT_DEFAULT)) {
+ ds.getInputDescriptor().setHistoryText(TezUtils.convertToHistoryText(conf));
+ }
+
+ return ds;
}
private DataSourceDescriptor createCustomDataSource() throws IOException {
@@ -279,6 +288,12 @@ public class MRInput extends MRInputBase {
DataSourceDescriptor ds = DataSourceDescriptor
.create(InputDescriptor.create(inputClassName).setUserPayload(payload),
customInitializerDescriptor, null);
+
+ if (conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT,
+ TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT_DEFAULT)) {
+ ds.getInputDescriptor().setHistoryText(TezUtils.convertToHistoryText(conf));
+ }
+
if (uris != null) {
ds.addURIsForCredentials(uris);
}
@@ -297,9 +312,16 @@ public class MRInput extends MRInputBase {
} else {
payload = MRInputHelpersInternal.createMRInputPayload(conf, null);
}
+
DataSourceDescriptor ds = DataSourceDescriptor.create(
InputDescriptor.create(inputClassName).setUserPayload(payload),
InputInitializerDescriptor.create(MRInputAMSplitGenerator.class.getName()), null);
+
+ if (conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT,
+ TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT_DEFAULT)) {
+ ds.getInputDescriptor().setHistoryText(TezUtils.convertToHistoryText(conf));
+ }
+
if (uris != null) {
ds.addURIsForCredentials(uris);
}
http://git-wip-us.apache.org/repos/asf/tez/blob/0daf2ba1/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java
index 421fc8c..94a3c1f 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java
@@ -66,6 +66,7 @@ import org.apache.tez.runtime.api.Event;
import org.apache.tez.runtime.api.Output;
import org.apache.tez.runtime.api.OutputContext;
import org.apache.tez.runtime.library.api.KeyValueWriter;
+import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
/**
* {@link MROutput} is an {@link Output} which allows key/values pairs
@@ -183,6 +184,11 @@ public class MROutput extends AbstractLogicalOutput {
OutputDescriptor.create(outputClassName).setUserPayload(createUserPayload()),
(doCommit ? OutputCommitterDescriptor.create(
MROutputCommitter.class.getName()) : null), null);
+ if (conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT,
+ TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT_DEFAULT)) {
+ ds.getOutputDescriptor().setHistoryText(TezUtils.convertToHistoryText(conf));
+ }
+
if (uris != null) {
ds.addURIsForCredentials(uris);
}
http://git-wip-us.apache.org/repos/asf/tez/blob/0daf2ba1/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java
index 0108c26..99cb441 100644
--- a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java
+++ b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java
@@ -103,9 +103,11 @@ public class ATSHistoryLoggingService extends HistoryLoggingService {
// Log the size of the event-queue every so often.
if (eventCounter != 0 && eventCounter % 1000 == 0) {
- LOG.info("Event queue stats"
- + ", eventsProcessedSinceLastUpdate=" + eventsProcessed
- + ", eventQueueSize=" + eventQueue.size());
+ if (eventsProcessed != 0 && !events.isEmpty()) {
+ LOG.info("Event queue stats"
+ + ", eventsProcessedSinceLastUpdate=" + eventsProcessed
+ + ", eventQueueSize=" + eventQueue.size());
+ }
eventCounter = 0;
eventsProcessed = 0;
} else {
http://git-wip-us.apache.org/repos/asf/tez/blob/0daf2ba1/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/TezRuntimeConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/TezRuntimeConfiguration.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/TezRuntimeConfiguration.java
index cb61109..3c0f11c 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/TezRuntimeConfiguration.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/TezRuntimeConfiguration.java
@@ -294,6 +294,13 @@ public class TezRuntimeConfiguration {
// TODO TEZ-1233 - allow this property to be set per vertex
// TODO TEZ-1231 - move these properties out since they are not relevant for Inputs / Outputs
+ /**
+ * Value: Boolean
+ * Whether to publish configuration information to History logger. Default false.
+ */
+ public static final String TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT =
+ TEZ_RUNTIME_PREFIX + "convert.user-payload.to.history-text";
+ public static final boolean TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT_DEFAULT = false;
@Unstable
@Private
@@ -345,6 +352,7 @@ public class TezRuntimeConfiguration {
tezRuntimeKeys.add(TEZ_RUNTIME_RECORDS_BEFORE_PROGRESS);
tezRuntimeKeys.add(TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH);
tezRuntimeKeys.add(TEZ_RUNTIME_OPTIMIZE_SHARED_FETCH);
+ tezRuntimeKeys.add(TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT);
defaultConf.addResource("core-default.xml");
defaultConf.addResource("core-site.xml");
http://git-wip-us.apache.org/repos/asf/tez/blob/0daf2ba1/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/HadoopKeyValuesBasedBaseEdgeConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/HadoopKeyValuesBasedBaseEdgeConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/HadoopKeyValuesBasedBaseEdgeConfig.java
index 31eb686..0692bac 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/HadoopKeyValuesBasedBaseEdgeConfig.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/HadoopKeyValuesBasedBaseEdgeConfig.java
@@ -46,6 +46,19 @@ abstract class HadoopKeyValuesBasedBaseEdgeConfig {
*/
public abstract UserPayload getInputPayload();
+
+ /**
+ * Get the history text for the configured Output
+ * @return output configuration as a string in json format
+ */
+ public abstract String getOutputHistoryText();
+
+ /**
+ * Get the history text for the configured Input
+ * @return input configuration as a string in json format
+ */
+ public abstract String getInputHistoryText();
+
/**
* Get the input class name
* @return the input class name
http://git-wip-us.apache.org/repos/asf/tez/blob/0daf2ba1/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedGroupedKVInputConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedGroupedKVInputConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedGroupedKVInputConfig.java
index 0e28399..888f61a 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedGroupedKVInputConfig.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedGroupedKVInputConfig.java
@@ -259,6 +259,15 @@ public class OrderedGroupedKVInputConfig {
}
}
+ @InterfaceAudience.Private
+ String toHistoryText() {
+ if (conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT,
+ TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT_DEFAULT)) {
+ return TezUtils.convertToHistoryText(conf);
+ }
+ return null;
+ }
+
public String getInputClassName() {
return inputClassName;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/0daf2ba1/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java
index 0f10cf1..e6cc2c2 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java
@@ -104,6 +104,16 @@ public class OrderedPartitionedKVEdgeConfig
}
@Override
+ public String getOutputHistoryText() {
+ return outputConf.toHistoryText();
+ }
+
+ @Override
+ public String getInputHistoryText() {
+ return inputConf.toHistoryText();
+ }
+
+ @Override
public String getInputClassName() {
return inputConf.getInputClassName();
}
@@ -123,6 +133,7 @@ public class OrderedPartitionedKVEdgeConfig
getOutputClassName()).setUserPayload(getOutputPayload()),
InputDescriptor.create(
getInputClassName()).setUserPayload(getInputPayload()));
+ Utils.setEdgePropertyHistoryText(this, edgeProperty);
return edgeProperty;
}
@@ -140,6 +151,7 @@ public class OrderedPartitionedKVEdgeConfig
EdgeProperty.SchedulingType.SEQUENTIAL,
OutputDescriptor.create(getOutputClassName()).setUserPayload(getOutputPayload()),
InputDescriptor.create(getInputClassName()).setUserPayload(getInputPayload()));
+ Utils.setEdgePropertyHistoryText(this, edgeProperty);
return edgeProperty;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/0daf2ba1/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVOutputConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVOutputConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVOutputConfig.java
index 1c4cca0..5437620 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVOutputConfig.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVOutputConfig.java
@@ -189,6 +189,15 @@ public class OrderedPartitionedKVOutputConfig {
}
}
+ @InterfaceAudience.Private
+ String toHistoryText() {
+ if (conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT,
+ TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT_DEFAULT)) {
+ return TezUtils.convertToHistoryText(conf);
+ }
+ return null;
+ }
+
public static Builder newBuilder(String keyClass, String valueClass, String partitionerClassName) {
return newBuilder(keyClass, valueClass, partitionerClassName, null);
}
http://git-wip-us.apache.org/repos/asf/tez/blob/0daf2ba1/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java
index 6eb1d6a..28a7503 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java
@@ -83,6 +83,16 @@ public class UnorderedKVEdgeConfig extends HadoopKeyValuesBasedBaseEdgeConfig {
}
@Override
+ public String getOutputHistoryText() {
+ return outputConf.toHistoryText();
+ }
+
+ @Override
+ public String getInputHistoryText() {
+ return inputConf.toHistoryText();
+ }
+
+ @Override
public String getInputClassName() {
return UnorderedKVInput.class.getName();
}
@@ -104,6 +114,7 @@ public class UnorderedKVEdgeConfig extends HadoopKeyValuesBasedBaseEdgeConfig {
getOutputClassName()).setUserPayload(getOutputPayload()),
InputDescriptor.create(
getInputClassName()).setUserPayload(getInputPayload()));
+ Utils.setEdgePropertyHistoryText(this, edgeProperty);
return edgeProperty;
}
@@ -124,6 +135,7 @@ public class UnorderedKVEdgeConfig extends HadoopKeyValuesBasedBaseEdgeConfig {
getOutputClassName()).setUserPayload(getOutputPayload()),
InputDescriptor.create(
getInputClassName()).setUserPayload(getInputPayload()));
+ Utils.setEdgePropertyHistoryText(this, edgeProperty);
return edgeProperty;
}
@@ -141,6 +153,7 @@ public class UnorderedKVEdgeConfig extends HadoopKeyValuesBasedBaseEdgeConfig {
EdgeProperty.SchedulingType.SEQUENTIAL,
OutputDescriptor.create(getOutputClassName()).setUserPayload(getOutputPayload()),
InputDescriptor.create(getInputClassName()).setUserPayload(getInputPayload()));
+ Utils.setEdgePropertyHistoryText(this, edgeProperty);
return edgeProperty;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/0daf2ba1/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVInputConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVInputConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVInputConfig.java
index bd3750e..e99b0bf 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVInputConfig.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVInputConfig.java
@@ -179,6 +179,15 @@ public class UnorderedKVInputConfig {
}
}
+ @InterfaceAudience.Private
+ String toHistoryText() {
+ if (conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT,
+ TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT_DEFAULT)) {
+ return TezUtils.convertToHistoryText(conf);
+ }
+ return null;
+ }
+
public static Builder newBuilder(String keyClass, String valueClass) {
return new Builder(keyClass, valueClass);
}
http://git-wip-us.apache.org/repos/asf/tez/blob/0daf2ba1/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVOutputConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVOutputConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVOutputConfig.java
index fdeffb3..30df2e3 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVOutputConfig.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVOutputConfig.java
@@ -125,6 +125,15 @@ public class UnorderedKVOutputConfig {
}
}
+ @InterfaceAudience.Private
+ String toHistoryText() {
+ if (conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT,
+ TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT_DEFAULT)) {
+ return TezUtils.convertToHistoryText(conf);
+ }
+ return null;
+ }
+
public static Builder newBuilder(String keyClass, String valClass) {
return new Builder(keyClass, valClass);
}
http://git-wip-us.apache.org/repos/asf/tez/blob/0daf2ba1/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java
index 30585bd..8ac7d65 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java
@@ -108,6 +108,16 @@ public class UnorderedPartitionedKVEdgeConfig
}
@Override
+ public String getOutputHistoryText() {
+ return outputConf.toHistoryText();
+ }
+
+ @Override
+ public String getInputHistoryText() {
+ return inputConf.toHistoryText();
+ }
+
+ @Override
public String getInputClassName() {
return UnorderedKVInput.class.getName();
}
@@ -129,6 +139,7 @@ public class UnorderedPartitionedKVEdgeConfig
getOutputClassName()).setUserPayload(getOutputPayload()),
InputDescriptor.create(
getInputClassName()).setUserPayload(getInputPayload()));
+ Utils.setEdgePropertyHistoryText(this, edgeProperty);
return edgeProperty;
}
@@ -146,6 +157,7 @@ public class UnorderedPartitionedKVEdgeConfig
EdgeProperty.SchedulingType.SEQUENTIAL,
OutputDescriptor.create(getOutputClassName()).setUserPayload(getOutputPayload()),
InputDescriptor.create(getInputClassName()).setUserPayload(getInputPayload()));
+ Utils.setEdgePropertyHistoryText(this, edgeProperty);
return edgeProperty;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/0daf2ba1/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVOutputConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVOutputConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVOutputConfig.java
index fae9d0f..0c8cd0d 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVOutputConfig.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVOutputConfig.java
@@ -139,6 +139,14 @@ public class UnorderedPartitionedKVOutputConfig {
}
}
+ @InterfaceAudience.Private
+ String toHistoryText() {
+ if (conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT,
+ TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT_DEFAULT)) {
+ return TezUtils.convertToHistoryText(conf);
+ }
+ return null;
+ }
public static Builder newBuilder(String keyClass, String valClass, String partitionerClassName) {
return newBuilder(keyClass, valClass, partitionerClassName, null);
http://git-wip-us.apache.org/repos/asf/tez/blob/0daf2ba1/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/Utils.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/Utils.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/Utils.java
new file mode 100644
index 0000000..c1b44a2
--- /dev/null
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/Utils.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.runtime.library.conf;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.tez.dag.api.EdgeProperty;
+
+@Private
+class Utils {
+
+ /**
+ * Modify the EdgeProperty to set the history text if available
+ * @param edgeConfig Edge config
+ * @param edgeProperty Edge property to be be modified
+ */
+ static void setEdgePropertyHistoryText(HadoopKeyValuesBasedBaseEdgeConfig edgeConfig,
+ EdgeProperty edgeProperty) {
+ String inputHistoryText = edgeConfig.getInputHistoryText();
+ if (inputHistoryText != null) {
+ edgeProperty.getEdgeDestination().setHistoryText(inputHistoryText);
+ }
+ String outputHistoryText = edgeConfig.getOutputHistoryText();
+ if (outputHistoryText != null) {
+ edgeProperty.getEdgeSource().setHistoryText(outputHistoryText);
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/0daf2ba1/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedKVInput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedKVInput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedKVInput.java
index 4231442..f46f8f7 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedKVInput.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedKVInput.java
@@ -334,6 +334,7 @@ public class OrderedGroupedKVInput extends AbstractLogicalInput {
confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC);
confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_SECONDARY_COMPARATOR_CLASS);
confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT);
confKeys.add(TezConfiguration.TEZ_COUNTERS_MAX);
confKeys.add(TezConfiguration.TEZ_COUNTERS_GROUP_NAME_MAX_LENGTH);
confKeys.add(TezConfiguration.TEZ_COUNTERS_COUNTER_NAME_MAX_LENGTH);
http://git-wip-us.apache.org/repos/asf/tez/blob/0daf2ba1/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/UnorderedKVInput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/UnorderedKVInput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/UnorderedKVInput.java
index 368c988..0d02cb3 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/UnorderedKVInput.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/UnorderedKVInput.java
@@ -249,6 +249,7 @@ public class UnorderedKVInput extends AbstractLogicalInput {
confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS);
confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC);
confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT);
confKeys.add(TezConfiguration.TEZ_COUNTERS_MAX);
confKeys.add(TezConfiguration.TEZ_COUNTERS_GROUP_NAME_MAX_LENGTH);
confKeys.add(TezConfiguration.TEZ_COUNTERS_COUNTER_NAME_MAX_LENGTH);
http://git-wip-us.apache.org/repos/asf/tez/blob/0daf2ba1/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OrderedPartitionedKVOutput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OrderedPartitionedKVOutput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OrderedPartitionedKVOutput.java
index 70d5b78..b3290a5 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OrderedPartitionedKVOutput.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OrderedPartitionedKVOutput.java
@@ -236,6 +236,7 @@ public class OrderedPartitionedKVOutput extends AbstractLogicalOutput {
confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS);
confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC);
confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_EMPTY_PARTITION_INFO_VIA_EVENTS_ENABLED);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT);
confKeys.add(TezConfiguration.TEZ_COUNTERS_MAX);
confKeys.add(TezConfiguration.TEZ_COUNTERS_GROUP_NAME_MAX_LENGTH);
confKeys.add(TezConfiguration.TEZ_COUNTERS_COUNTER_NAME_MAX_LENGTH);
http://git-wip-us.apache.org/repos/asf/tez/blob/0daf2ba1/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedKVOutput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedKVOutput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedKVOutput.java
index 6a84e61..7bfc397 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedKVOutput.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedKVOutput.java
@@ -179,6 +179,7 @@ public class UnorderedKVOutput extends AbstractLogicalOutput {
confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS);
confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC);
confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_EMPTY_PARTITION_INFO_VIA_EVENTS_ENABLED);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT);
confKeys.add(TezConfiguration.TEZ_COUNTERS_MAX);
confKeys.add(TezConfiguration.TEZ_COUNTERS_GROUP_NAME_MAX_LENGTH);
confKeys.add(TezConfiguration.TEZ_COUNTERS_COUNTER_NAME_MAX_LENGTH);
http://git-wip-us.apache.org/repos/asf/tez/blob/0daf2ba1/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedPartitionedKVOutput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedPartitionedKVOutput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedPartitionedKVOutput.java
index 11e6849..1e39535 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedPartitionedKVOutput.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedPartitionedKVOutput.java
@@ -119,6 +119,7 @@ public class UnorderedPartitionedKVOutput extends AbstractLogicalOutput {
confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS);
confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC);
confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_EMPTY_PARTITION_INFO_VIA_EVENTS_ENABLED);
+ confKeys.add(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT);
confKeys.add(TezConfiguration.TEZ_COUNTERS_MAX);
confKeys.add(TezConfiguration.TEZ_COUNTERS_GROUP_NAME_MAX_LENGTH);
confKeys.add(TezConfiguration.TEZ_COUNTERS_COUNTER_NAME_MAX_LENGTH);
http://git-wip-us.apache.org/repos/asf/tez/blob/0daf2ba1/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVEdgeConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVEdgeConfig.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVEdgeConfig.java
index c595a9d..35fd3cb 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVEdgeConfig.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVEdgeConfig.java
@@ -21,9 +21,11 @@
package org.apache.tez.runtime.library.conf;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
import java.util.HashMap;
import java.util.Map;
@@ -31,6 +33,8 @@ import java.util.Map;
import com.google.common.collect.Maps;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.tez.dag.api.EdgeManagerPluginDescriptor;
+import org.apache.tez.dag.api.EdgeProperty;
import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
import org.junit.Test;
@@ -60,7 +64,7 @@ public class TestOrderedPartitionedKVEdgeConfig {
}
}
- @Test
+ @Test (timeout=2000)
public void testDefaultConfigsUsed() {
OrderedPartitionedKVEdgeConfig.Builder builder = OrderedPartitionedKVEdgeConfig
.newBuilder("KEY", "VALUE", "PARTITIONER");
@@ -86,7 +90,7 @@ public class TestOrderedPartitionedKVEdgeConfig {
inputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, ""));
}
- @Test
+ @Test (timeout=2000)
public void testSpecificIOConfs() {
// Ensures that Output and Input confs are not mixed.
OrderedPartitionedKVEdgeConfig.Builder builder = OrderedPartitionedKVEdgeConfig
@@ -109,7 +113,7 @@ public class TestOrderedPartitionedKVEdgeConfig {
inputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, "DEFAULT"));
}
- @Test
+ @Test (timeout=2000)
public void tetCommonConf() {
Configuration fromConf = new Configuration(false);
@@ -179,7 +183,7 @@ public class TestOrderedPartitionedKVEdgeConfig {
}
- @Test
+ @Test (timeout=2000)
public void testSetters() {
Map<String, String> comparatorConf = Maps.newHashMap();
comparatorConf.put("comparator.test.key", "comparatorValue");
@@ -244,7 +248,7 @@ public class TestOrderedPartitionedKVEdgeConfig {
}
- @Test
+ @Test (timeout=2000)
public void testSerialization() {
OrderedPartitionedKVEdgeConfig.Builder builder = OrderedPartitionedKVEdgeConfig
.newBuilder("KEY", "VALUE", "PARTITIONER")
@@ -294,4 +298,35 @@ public class TestOrderedPartitionedKVEdgeConfig {
inputConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS,
false));
}
+
+ private void checkHistoryText(String historyText) {
+ assertNotNull(historyText);
+ assertTrue(historyText.contains(
+ TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT));
+ }
+
+ @Test (timeout=2000)
+ public void testHistoryText() {
+ OrderedPartitionedKVEdgeConfig.Builder builder =
+ OrderedPartitionedKVEdgeConfig.newBuilder("KEY", "VALUE", "PARTITIONER");
+ Configuration fromConf = new Configuration(false);
+ fromConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT,
+ true);
+ builder.setFromConfiguration(fromConf);
+
+ OrderedPartitionedKVEdgeConfig kvEdgeConfig = builder.build();
+ checkHistoryText(kvEdgeConfig.getInputHistoryText());
+ checkHistoryText(kvEdgeConfig.getOutputHistoryText());
+
+ EdgeProperty defaultEdgeProperty = builder.build().createDefaultEdgeProperty();
+ checkHistoryText(defaultEdgeProperty.getEdgeDestination().getHistoryText());
+ checkHistoryText(defaultEdgeProperty.getEdgeSource().getHistoryText());
+
+ EdgeManagerPluginDescriptor descriptor = mock(EdgeManagerPluginDescriptor.class);
+ EdgeProperty edgeProperty = builder.build().createDefaultCustomEdgeProperty(descriptor);
+ checkHistoryText(edgeProperty.getEdgeDestination().getHistoryText());
+ checkHistoryText(edgeProperty.getEdgeSource().getHistoryText());
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/tez/blob/0daf2ba1/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVEdgeConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVEdgeConfig.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVEdgeConfig.java
index fab1e94..0ef1cc5 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVEdgeConfig.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVEdgeConfig.java
@@ -21,21 +21,25 @@
package org.apache.tez.runtime.library.conf;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.tez.dag.api.EdgeManagerPluginDescriptor;
+import org.apache.tez.dag.api.EdgeProperty;
import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
import org.junit.Test;
public class TestUnorderedKVEdgeConfig {
- @Test
+ @Test (timeout=2000)
public void testNullParams() {
try {
UnorderedKVEdgeConfig.newBuilder(null, "VALUE");
@@ -52,7 +56,7 @@ public class TestUnorderedKVEdgeConfig {
}
}
- @Test
+ @Test (timeout=2000)
public void testDefaultConfigsUsed() {
UnorderedKVEdgeConfig.Builder builder =
UnorderedKVEdgeConfig.newBuilder("KEY", "VALUE");
@@ -85,7 +89,7 @@ public class TestUnorderedKVEdgeConfig {
("SerClass2,SerClass1"));
}
- @Test
+ @Test (timeout=2000)
public void testSpecificIOConfs() {
// Ensures that Output and Input confs are not mixed.
UnorderedKVEdgeConfig.Builder builder =
@@ -109,7 +113,7 @@ public class TestUnorderedKVEdgeConfig {
inputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, "DEFAULT"));
}
- @Test
+ @Test (timeout=2000)
public void tetCommonConf() {
Configuration fromConf = new Configuration(false);
@@ -170,4 +174,40 @@ public class TestUnorderedKVEdgeConfig {
assertEquals("fs", inputConf.get("fs.shouldExist"));
}
+
+ private void checkHistoryText(String historyText) {
+ assertNotNull(historyText);
+ assertTrue(historyText.contains(
+ TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT));
+ }
+
+ @Test (timeout=2000)
+ public void testHistoryText() {
+ UnorderedKVEdgeConfig.Builder builder = UnorderedKVEdgeConfig.newBuilder("KEY", "VALUE");
+ Configuration fromConf = new Configuration(false);
+ fromConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT,
+ true);
+ builder.setFromConfiguration(fromConf);
+
+ UnorderedKVEdgeConfig kvEdgeConfig = builder.build();
+
+ checkHistoryText(kvEdgeConfig.getInputHistoryText());
+ checkHistoryText(kvEdgeConfig.getOutputHistoryText());
+
+ EdgeProperty defaultEdgeProperty = builder.build().createDefaultBroadcastEdgeProperty();
+ checkHistoryText(defaultEdgeProperty.getEdgeDestination().getHistoryText());
+ checkHistoryText(defaultEdgeProperty.getEdgeSource().getHistoryText());
+
+ defaultEdgeProperty = builder.build().createDefaultOneToOneEdgeProperty();
+ checkHistoryText(defaultEdgeProperty.getEdgeDestination().getHistoryText());
+ checkHistoryText(defaultEdgeProperty.getEdgeSource().getHistoryText());
+
+ EdgeManagerPluginDescriptor descriptor = mock(EdgeManagerPluginDescriptor.class);
+ EdgeProperty edgeProperty = builder.build().createDefaultCustomEdgeProperty(descriptor);
+ checkHistoryText(edgeProperty.getEdgeDestination().getHistoryText());
+ checkHistoryText(edgeProperty.getEdgeSource().getHistoryText());
+
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/tez/blob/0daf2ba1/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVEdgeConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVEdgeConfig.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVEdgeConfig.java
index b8b86e5..fff8efd 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVEdgeConfig.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVEdgeConfig.java
@@ -21,21 +21,25 @@
package org.apache.tez.runtime.library.conf;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.tez.dag.api.EdgeManagerPluginDescriptor;
+import org.apache.tez.dag.api.EdgeProperty;
import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
import org.junit.Test;
public class TestUnorderedPartitionedKVEdgeConfig {
- @Test
+ @Test (timeout=2000)
public void testNullParams() {
try {
UnorderedPartitionedKVEdgeConfig.newBuilder(null, "VALUE", "PARTITIONER");
@@ -59,7 +63,7 @@ public class TestUnorderedPartitionedKVEdgeConfig {
}
}
- @Test
+ @Test (timeout=2000)
public void testDefaultConfigsUsed() {
UnorderedPartitionedKVEdgeConfig.Builder builder =
UnorderedPartitionedKVEdgeConfig.newBuilder("KEY", "VALUE", "PARTITIONER");
@@ -92,7 +96,7 @@ public class TestUnorderedPartitionedKVEdgeConfig {
("SerClass2,SerClass1"));
}
- @Test
+ @Test (timeout=2000)
public void testSpecificIOConfs() {
// Ensures that Output and Input confs are not mixed.
UnorderedPartitionedKVEdgeConfig.Builder builder =
@@ -116,7 +120,7 @@ public class TestUnorderedPartitionedKVEdgeConfig {
inputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, "DEFAULT"));
}
- @Test
+ @Test (timeout=2000)
public void tetCommonConf() {
Configuration fromConf = new Configuration(false);
@@ -185,4 +189,36 @@ public class TestUnorderedPartitionedKVEdgeConfig {
assertEquals("fs", inputConf.get("fs.shouldExist"));
}
+
+ private void checkHistoryText(String historyText) {
+ assertNotNull(historyText);
+ assertTrue(historyText.contains(
+ TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT));
+ }
+
+ @Test (timeout=2000)
+ public void testHistoryText() {
+ UnorderedPartitionedKVEdgeConfig.Builder builder =
+ UnorderedPartitionedKVEdgeConfig.newBuilder("KEY", "VALUE", "PARTITIONER");
+ Configuration fromConf = new Configuration(false);
+ fromConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT,
+ true);
+ builder.setFromConfiguration(fromConf);
+
+ UnorderedPartitionedKVEdgeConfig kvEdgeConfig = builder.build();
+
+ checkHistoryText(kvEdgeConfig.getInputHistoryText());
+ checkHistoryText(kvEdgeConfig.getOutputHistoryText());
+
+ EdgeProperty defaultEdgeProperty = builder.build().createDefaultEdgeProperty();
+ checkHistoryText(defaultEdgeProperty.getEdgeDestination().getHistoryText());
+ checkHistoryText(defaultEdgeProperty.getEdgeSource().getHistoryText());
+
+ EdgeManagerPluginDescriptor descriptor = mock(EdgeManagerPluginDescriptor.class);
+ EdgeProperty edgeProperty = builder.build().createDefaultCustomEdgeProperty(descriptor);
+ checkHistoryText(edgeProperty.getEdgeDestination().getHistoryText());
+ checkHistoryText(edgeProperty.getEdgeSource().getHistoryText());
+
+ }
+
}
http://git-wip-us.apache.org/repos/asf/tez/blob/0daf2ba1/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java b/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
index 9ca2762..86dcc64 100644
--- a/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
+++ b/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
@@ -77,6 +77,7 @@ import org.apache.tez.common.TezUtils;
import org.apache.tez.common.counters.FileSystemCounter;
import org.apache.tez.common.counters.TaskCounter;
import org.apache.tez.dag.api.DAG;
+import org.apache.tez.dag.api.DataSinkDescriptor;
import org.apache.tez.dag.api.DataSourceDescriptor;
import org.apache.tez.dag.api.Edge;
import org.apache.tez.dag.api.EdgeProperty;
@@ -118,6 +119,7 @@ import org.apache.tez.mapreduce.protos.MRRuntimeProtos.MRInputUserPayloadProto;
import org.apache.tez.runtime.api.Event;
import org.apache.tez.runtime.api.InputInitializer;
import org.apache.tez.runtime.api.InputInitializerContext;
+import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
import org.apache.tez.runtime.library.input.OrderedGroupedInputLegacy;
import org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput;
import org.apache.tez.runtime.library.processor.SleepProcessor;
@@ -691,8 +693,12 @@ public class TestMRRJobsDAGApi {
Vertex stage3Vertex = Vertex.create("reduce", ProcessorDescriptor.create(
ReduceProcessor.class.getName()).setUserPayload(stage3Payload),
1, Resource.newInstance(256, 1));
- stage3Vertex.addDataSink("MROutput",
- MROutputLegacy.createConfigBuilder(stage3Conf, NullOutputFormat.class).build());
+ stage3Conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT,
+ true);
+ DataSinkDescriptor dataSinkDescriptor =
+ MROutputLegacy.createConfigBuilder(stage3Conf, NullOutputFormat.class).build();
+ Assert.assertFalse(dataSinkDescriptor.getOutputDescriptor().getHistoryText().isEmpty());
+ stage3Vertex.addDataSink("MROutput", dataSinkDescriptor);
// TODO env, resources
[26/50] [abbrv] tez git commit: TEZ-1627. Remove OUTPUT_CONSUMABLE
and related Event in TaskAttemptImpl (zjffdu)
Posted by je...@apache.org.
TEZ-1627. Remove OUTPUT_CONSUMABLE and related Event in TaskAttemptImpl (zjffdu)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/31c1de6d
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/31c1de6d
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/31c1de6d
Branch: refs/heads/TEZ-8
Commit: 31c1de6daa7b6eff1a32a8d10999680084085fc1
Parents: 71ca74e
Author: Jeff Zhang <zj...@apache.org>
Authored: Fri Nov 7 10:20:37 2014 +0800
Committer: Jeff Zhang <zj...@apache.org>
Committed: Fri Nov 7 10:20:37 2014 +0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../java/org/apache/tez/dag/app/dag/Task.java | 19 -----
.../dag/app/dag/TaskAttemptStateInternal.java | 1 -
.../dag/app/dag/event/TaskAttemptEventType.java | 1 -
.../tez/dag/app/dag/event/TaskEventType.java | 1 -
.../tez/dag/app/dag/impl/TaskAttemptImpl.java | 72 -----------------
.../apache/tez/dag/app/dag/impl/TaskImpl.java | 84 --------------------
7 files changed, 1 insertion(+), 178 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/31c1de6d/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 2e58f0a..8fad485 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -8,6 +8,7 @@ INCOMPATIBLE CHANGES
ALL CHANGES:
TEZ-1733. TezMerger should sort FileChunks on size when merging
TEZ-1738. Tez tfile parser for log parsing
+ TEZ-1627. Remove OUTPUT_CONSUMABLE and related Event in TaskAttemptImpl
Release 0.5.3: Unreleased
http://git-wip-us.apache.org/repos/asf/tez/blob/31c1de6d/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java
index b5fa45f..98a85cf 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java
@@ -55,25 +55,6 @@ public interface Task {
* @return whether the attempt's output can be committed or not.
*/
boolean canCommit(TezTaskAttemptID taskAttemptID);
-
-
- /**
- * Do the running tasks need to stick around after they're done processing and
- * generating output. Required for tasks which have custom output handling
- * such as in-memory shuffle.
- *
- * @return whether the task needs to stick around.
- */
- boolean needsWaitAfterOutputConsumable();
-
- /**
- * Get the attempt id which has reported in as output ready. null if not
- * applicable.
- *
- * @return the attempt id which has reported in as output ready. null if not
- * applicable.
- */
- TezTaskAttemptID getOutputConsumableAttempt();
public Vertex getVertex();
http://git-wip-us.apache.org/repos/asf/tez/blob/31c1de6d/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttemptStateInternal.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttemptStateInternal.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttemptStateInternal.java
index a49c2a3..8d0d83e 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttemptStateInternal.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttemptStateInternal.java
@@ -29,7 +29,6 @@ public enum TaskAttemptStateInternal {
NEW,
START_WAIT,
RUNNING,
- OUTPUT_CONSUMABLE,
KILL_IN_PROGRESS,
FAIL_IN_PROGRESS,
KILLED,
http://git-wip-us.apache.org/repos/asf/tez/blob/31c1de6d/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventType.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventType.java
index e7db8d1..c8eec1b 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventType.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventType.java
@@ -30,7 +30,6 @@ public enum TaskAttemptEventType {
TA_STARTED_REMOTELY,
TA_STATUS_UPDATE,
TA_DIAGNOSTICS_UPDATE,
- TA_OUTPUT_CONSUMABLE, // TODO History event to indicate this ?
TA_COMMIT_PENDING,
TA_DONE,
TA_FAILED,
http://git-wip-us.apache.org/repos/asf/tez/blob/31c1de6d/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventType.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventType.java
index fe4dd5a..baec5f0 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventType.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventType.java
@@ -34,7 +34,6 @@ public enum TaskEventType {
//Producer:TaskAttempt
T_ATTEMPT_LAUNCHED,
- T_ATTEMPT_OUTPUT_CONSUMABLE,
T_ATTEMPT_FAILED,
T_ATTEMPT_SUCCEEDED,
T_ATTEMPT_KILLED,
http://git-wip-us.apache.org/repos/asf/tez/blob/31c1de6d/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
index 45ca543..deaba42 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
@@ -236,10 +236,6 @@ public class TaskAttemptImpl implements TaskAttempt,
TaskAttemptStateInternal.RUNNING,
TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
- .addTransition(TaskAttemptStateInternal.RUNNING,
- TaskAttemptStateInternal.OUTPUT_CONSUMABLE,
- TaskAttemptEventType.TA_OUTPUT_CONSUMABLE,
- new OutputConsumableTransition())
// Optional, may not come in for all tasks.
.addTransition(TaskAttemptStateInternal.RUNNING,
TaskAttemptStateInternal.SUCCEEDED, TaskAttemptEventType.TA_DONE,
@@ -279,57 +275,6 @@ public class TaskAttemptImpl implements TaskAttempt,
TaskAttemptEventType.TA_OUTPUT_FAILED,
new OutputReportedFailedTransition())
- .addTransition(TaskAttemptStateInternal.OUTPUT_CONSUMABLE,
- TaskAttemptStateInternal.OUTPUT_CONSUMABLE,
- TaskAttemptEventType.TA_STATUS_UPDATE, STATUS_UPDATER)
- .addTransition(TaskAttemptStateInternal.OUTPUT_CONSUMABLE,
- TaskAttemptStateInternal.OUTPUT_CONSUMABLE,
- TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
- DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
- .addTransition(TaskAttemptStateInternal.OUTPUT_CONSUMABLE,
- TaskAttemptStateInternal.OUTPUT_CONSUMABLE,
- TaskAttemptEventType.TA_OUTPUT_CONSUMABLE)
- // Stuck RPC. The client retries in a loop.
- .addTransition(TaskAttemptStateInternal.OUTPUT_CONSUMABLE,
- TaskAttemptStateInternal.SUCCEEDED, TaskAttemptEventType.TA_DONE,
- new SucceededTransition())
- .addTransition(TaskAttemptStateInternal.OUTPUT_CONSUMABLE,
- TaskAttemptStateInternal.FAIL_IN_PROGRESS,
- TaskAttemptEventType.TA_FAILED,
- new TerminatedWhileRunningTransition(FAILED_HELPER))
- .addTransition(TaskAttemptStateInternal.OUTPUT_CONSUMABLE,
- TaskAttemptStateInternal.FAIL_IN_PROGRESS,
- TaskAttemptEventType.TA_TIMED_OUT,
- new TerminatedWhileRunningTransition(FAILED_HELPER))
- // TODO CREUSE Ensure TaskCompletionEvents are updated to reflect this.
- // Something needs to go out to the job.
- .addTransition(TaskAttemptStateInternal.OUTPUT_CONSUMABLE,
- TaskAttemptStateInternal.KILL_IN_PROGRESS,
- TaskAttemptEventType.TA_KILL_REQUEST,
- new TerminatedWhileRunningTransition(KILLED_HELPER))
- .addTransition(TaskAttemptStateInternal.OUTPUT_CONSUMABLE,
- TaskAttemptStateInternal.KILL_IN_PROGRESS,
- TaskAttemptEventType.TA_NODE_FAILED,
- new TerminatedWhileRunningTransition(KILLED_HELPER))
- .addTransition(TaskAttemptStateInternal.OUTPUT_CONSUMABLE,
- TaskAttemptStateInternal.FAIL_IN_PROGRESS,
- TaskAttemptEventType.TA_CONTAINER_TERMINATING,
- new TerminatedWhileRunningTransition(FAILED_HELPER))
- .addTransition(TaskAttemptStateInternal.OUTPUT_CONSUMABLE,
- TaskAttemptStateInternal.FAILED,
- TaskAttemptEventType.TA_CONTAINER_TERMINATED,
- new ContainerCompletedBeforeRunningTransition())
- .addTransition(TaskAttemptStateInternal.OUTPUT_CONSUMABLE,
- TaskAttemptStateInternal.KILLED,
- TaskAttemptEventType.TA_CONTAINER_TERMINATED_BY_SYSTEM,
- new ContainerCompletedBeforeRunningTransition(KILLED_HELPER))
- .addTransition(
- TaskAttemptStateInternal.OUTPUT_CONSUMABLE,
- EnumSet.of(TaskAttemptStateInternal.FAIL_IN_PROGRESS,
- TaskAttemptStateInternal.OUTPUT_CONSUMABLE),
- TaskAttemptEventType.TA_OUTPUT_FAILED,
- new OutputReportedFailedTransition())
-
.addTransition(TaskAttemptStateInternal.KILL_IN_PROGRESS,
TaskAttemptStateInternal.KILLED,
TaskAttemptEventType.TA_CONTAINER_TERMINATED,
@@ -344,7 +289,6 @@ public class TaskAttemptImpl implements TaskAttempt,
EnumSet.of(TaskAttemptEventType.TA_STARTED_REMOTELY,
TaskAttemptEventType.TA_CONTAINER_TERMINATED_BY_SYSTEM,
TaskAttemptEventType.TA_STATUS_UPDATE,
- TaskAttemptEventType.TA_OUTPUT_CONSUMABLE,
TaskAttemptEventType.TA_COMMIT_PENDING,
TaskAttemptEventType.TA_DONE, TaskAttemptEventType.TA_FAILED,
TaskAttemptEventType.TA_TIMED_OUT,
@@ -367,7 +311,6 @@ public class TaskAttemptImpl implements TaskAttempt,
EnumSet.of(TaskAttemptEventType.TA_STARTED_REMOTELY,
TaskAttemptEventType.TA_CONTAINER_TERMINATED_BY_SYSTEM,
TaskAttemptEventType.TA_STATUS_UPDATE,
- TaskAttemptEventType.TA_OUTPUT_CONSUMABLE,
TaskAttemptEventType.TA_COMMIT_PENDING,
TaskAttemptEventType.TA_DONE, TaskAttemptEventType.TA_FAILED,
TaskAttemptEventType.TA_TIMED_OUT,
@@ -387,7 +330,6 @@ public class TaskAttemptImpl implements TaskAttempt,
TaskAttemptEventType.TA_SCHEDULE,
TaskAttemptEventType.TA_CONTAINER_TERMINATED_BY_SYSTEM,
TaskAttemptEventType.TA_STATUS_UPDATE,
- TaskAttemptEventType.TA_OUTPUT_CONSUMABLE,
TaskAttemptEventType.TA_COMMIT_PENDING,
TaskAttemptEventType.TA_DONE, TaskAttemptEventType.TA_FAILED,
TaskAttemptEventType.TA_TIMED_OUT,
@@ -408,7 +350,6 @@ public class TaskAttemptImpl implements TaskAttempt,
TaskAttemptEventType.TA_SCHEDULE,
TaskAttemptEventType.TA_CONTAINER_TERMINATED_BY_SYSTEM,
TaskAttemptEventType.TA_STATUS_UPDATE,
- TaskAttemptEventType.TA_OUTPUT_CONSUMABLE,
TaskAttemptEventType.TA_COMMIT_PENDING,
TaskAttemptEventType.TA_DONE, TaskAttemptEventType.TA_FAILED,
TaskAttemptEventType.TA_TIMED_OUT,
@@ -762,7 +703,6 @@ public class TaskAttemptImpl implements TaskAttempt,
case START_WAIT:
return TaskAttemptState.STARTING;
case RUNNING:
- case OUTPUT_CONSUMABLE:
return TaskAttemptState.RUNNING;
case FAILED:
case FAIL_IN_PROGRESS:
@@ -1298,18 +1238,6 @@ public class TaskAttemptImpl implements TaskAttempt,
}
}
- protected static class OutputConsumableTransition implements
- SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
-
- @Override
- public void transition(TaskAttemptImpl ta, TaskAttemptEvent event) {
- //TaskAttemptEventOutputConsumable orEvent = (TaskAttemptEventOutputConsumable) event;
- //ta.shufflePort = orEvent.getOutputContext().getShufflePort();
- ta.sendEvent(new TaskEventTAUpdate(ta.attemptId,
- TaskEventType.T_ATTEMPT_OUTPUT_CONSUMABLE));
- }
- }
-
protected static class SucceededTransition implements
SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
@Override
http://git-wip-us.apache.org/repos/asf/tez/blob/31c1de6d/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
index 7bafb3b..4ded9be 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java
@@ -187,10 +187,6 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
// Transitions from RUNNING state
.addTransition(TaskStateInternal.RUNNING, TaskStateInternal.RUNNING,
TaskEventType.T_ATTEMPT_LAUNCHED) //more attempts may start later
- // This is an optional event.
- .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.RUNNING,
- TaskEventType.T_ATTEMPT_OUTPUT_CONSUMABLE,
- new AttemptProcessingCompleteTransition())
.addTransition(TaskStateInternal.RUNNING, TaskStateInternal.RUNNING,
TaskEventType.T_ADD_SPEC_ATTEMPT, new RedundantScheduleTransition())
.addTransition(TaskStateInternal.RUNNING, TaskStateInternal.SUCCEEDED,
@@ -229,12 +225,9 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
EnumSet.of(
TaskEventType.T_TERMINATE,
TaskEventType.T_ATTEMPT_LAUNCHED,
- TaskEventType.T_ATTEMPT_OUTPUT_CONSUMABLE,
TaskEventType.T_ADD_SPEC_ATTEMPT))
// Transitions from SUCCEEDED state
- // TODO May required different handling if OUTPUT_CONSUMABLE is one of
- // the stages. i.e. Task would only SUCCEED after all output consumed.
.addTransition(TaskStateInternal.SUCCEEDED, //only possible for map tasks
EnumSet.of(TaskStateInternal.SCHEDULED, TaskStateInternal.SUCCEEDED, TaskStateInternal.FAILED),
TaskEventType.T_ATTEMPT_FAILED, new TaskRetroactiveFailureTransition())
@@ -298,9 +291,6 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
*/
- private TezTaskAttemptID outputConsumableAttempt;
- private boolean outputConsumableAttemptSuccessSent = false;
-
//should be set to one which comes first
//saying COMMIT_PENDING
private TezTaskAttemptID commitAttempt;
@@ -750,30 +740,6 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
}
}
- // TODO remove hacky name lookup
- @Override
- public boolean needsWaitAfterOutputConsumable() {
- Vertex vertex = getVertex();
- ProcessorDescriptor processorDescriptor = vertex.getProcessorDescriptor();
- if (processorDescriptor != null &&
- processorDescriptor.getClassName().contains("InitialTaskWithInMemSort")) {
- return true;
- } else {
- return false;
- }
- }
-
-
- @Override
- public TezTaskAttemptID getOutputConsumableAttempt() {
- readLock.lock();
- try {
- return this.outputConsumableAttempt;
- } finally {
- readLock.unlock();
- }
- }
-
TaskAttemptImpl createAttempt(int attemptNumber) {
return new TaskAttemptImpl(getTaskId(), attemptNumber, eventHandler,
taskAttemptListener, conf, clock, taskHeartbeatHandler, appContext,
@@ -877,18 +843,6 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
private void sendTaskAttemptCompletionEvent(TezTaskAttemptID attemptId,
TaskAttemptStateInternal attemptState) {
- // raise the completion event only if the container is assigned
- // to nextAttemptNumber
- if (needsWaitAfterOutputConsumable()) {
- // An event may have been sent out during the OUTPUT_READY state itself.
- // Make sure the same event is not being sent out again.
- if (attemptId == outputConsumableAttempt
- && attemptState == TaskAttemptStateInternal.SUCCEEDED) {
- if (outputConsumableAttemptSuccessSent) {
- return;
- }
- }
- }
eventHandler.handle(new VertexEventTaskAttemptCompleted(attemptId, attemptState));
}
@@ -1037,38 +991,6 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
}
}
- private static class AttemptProcessingCompleteTransition implements
- SingleArcTransition<TaskImpl, TaskEvent> {
-
- @Override
- public void transition(TaskImpl task, TaskEvent event) {
- TaskEventTAUpdate taEvent = (TaskEventTAUpdate) event;
- TezTaskAttemptID attemptId = taEvent.getTaskAttemptID();
-
- if (task.outputConsumableAttempt == null) {
- task.sendTaskAttemptCompletionEvent(attemptId,
- TaskAttemptStateInternal.SUCCEEDED);
- task.outputConsumableAttempt = attemptId;
- task.outputConsumableAttemptSuccessSent = true;
- if (LOG.isDebugEnabled()) {
- LOG.debug("TezTaskAttemptID: " + attemptId
- + " set as the OUTPUT_READY attempt");
- }
- } else {
- // Nothing to do. This task will eventually be told to die, or will be
- // killed.
- if (LOG.isDebugEnabled()) {
- LOG.debug("TezTaskAttemptID: "
- + attemptId + " reporting OUTPUT_READY."
- + " Will be asked to die since another attempt "
- + task.outputConsumableAttempt + " already has output ready");
- }
- task.eventHandler.handle(new TaskAttemptEventKillRequest(attemptId,
- "Alternate attemptId already serving output"));
- }
-
- }
- }
private static class AttemptSucceededTransition
implements SingleArcTransition<TaskImpl, TaskEvent> {
@@ -1313,12 +1235,6 @@ public class TaskImpl implements Task, EventHandler<TaskEvent> {
castEvent.getTaskAttemptID().equals(task.commitAttempt)) {
task.commitAttempt = null;
}
- if (castEvent.getTaskAttemptID().equals(task.outputConsumableAttempt)) {
- task.outputConsumableAttempt = null;
- task.handleTaskAttemptCompletion(castEvent.getTaskAttemptID(),
- TaskAttemptStateInternal.FAILED);
- }
-
// The attempt would have informed the scheduler about it's failure
task.taskAttemptStatus.put(castEvent.getTaskAttemptID().getId(), true);
[15/50] [abbrv] tez git commit: TEZ-1726. Build broken against
Hadoop-2.6.0 due to change in NodeReport. (sseth)
Posted by je...@apache.org.
TEZ-1726. Build broken against Hadoop-2.6.0 due to change in NodeReport. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/b768c0f1
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/b768c0f1
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/b768c0f1
Branch: refs/heads/TEZ-8
Commit: b768c0f19be158ce05c62bcc9455b75c92ac8831
Parents: b66ff72
Author: Siddharth Seth <ss...@apache.org>
Authored: Fri Oct 31 01:23:06 2014 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Fri Oct 31 01:23:06 2014 -0700
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../tez/dag/app/rm/node/TestAMNodeTracker.java | 15 ++++++++++++---
2 files changed, 13 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/b768c0f1/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 9c28485..d43ccf7 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -96,6 +96,7 @@ ALL CHANGES:
TEZ-1698. Cut down on ResourceCalculatorProcessTree overheads in Tez.
TEZ-1703. addendum - fix flaky test.
TEZ-1725. Fix nanosecond to millis conversion in TezMxBeanResourceCalculator.
+ TEZ-1726. Build broken against Hadoop-2.6.0 due to change in NodeReport.
Release 0.5.1: 2014-10-02
http://git-wip-us.apache.org/repos/asf/tez/blob/b768c0f1/tez-dag/src/test/java/org/apache/tez/dag/app/rm/node/TestAMNodeTracker.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/node/TestAMNodeTracker.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/node/TestAMNodeTracker.java
index 8da2513..bc9ddee 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/node/TestAMNodeTracker.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/node/TestAMNodeTracker.java
@@ -322,9 +322,18 @@ public class TestAMNodeTracker {
}
private static NodeReport generateNodeReport(NodeId nodeId, NodeState nodeState) {
- NodeReport nodeReport = NodeReport.newInstance(nodeId, nodeState, nodeId.getHost() + ":3433",
- "/default-rack", Resource.newInstance(0, 0), Resource.newInstance(10240, 12), 10,
- nodeState.toString(), System.currentTimeMillis());
+ NodeReport nodeReport = mock(NodeReport.class);
+ doReturn(nodeId).when(nodeReport).getNodeId();
+ doReturn(nodeState).when(nodeReport).getNodeState();
+ String httpAddress = nodeId.getHost() + ":3433";
+ doReturn(httpAddress).when(nodeReport).getHttpAddress();
+ doReturn("/default-rack").when(nodeReport).getRackName();
+ doReturn(Resource.newInstance(0, 0)).when(nodeReport).getUsed();
+ doReturn(Resource.newInstance(10240, 12)).when(nodeReport).getCapability();
+ doReturn(10).when(nodeReport).getNumContainers();
+ doReturn(nodeState.toString()).when(nodeReport).getHealthReport();
+ long healthReportTime = System.currentTimeMillis();
+ doReturn(healthReportTime).when(nodeReport).getLastHealthReportTime();
return nodeReport;
}
}
[35/50] [abbrv] tez git commit: TEZ-1758. TezClient should provide
YARN diagnostics when the AM crashes (bikas)
Posted by je...@apache.org.
TEZ-1758. TezClient should provide YARN diagnostics when the AM crashes (bikas)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/a2d5768b
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/a2d5768b
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/a2d5768b
Branch: refs/heads/TEZ-8
Commit: a2d5768bc274684719dff64b46967264f3d18c0e
Parents: c6c08c1
Author: Bikas Saha <bi...@apache.org>
Authored: Sat Nov 8 11:59:14 2014 -0800
Committer: Bikas Saha <bi...@apache.org>
Committed: Sat Nov 8 11:59:14 2014 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../java/org/apache/tez/client/TezClient.java | 16 ++++-
.../org/apache/tez/client/TezClientUtils.java | 9 ++-
.../tez/dag/api/client/DAGClientImpl.java | 12 ----
.../org/apache/tez/client/TestTezClient.java | 65 +++++++++++++++++++-
5 files changed, 85 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/a2d5768b/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index da457df..34ffc84 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -13,6 +13,7 @@ ALL CHANGES:
Release 0.5.3: Unreleased
ALL CHANGES:
+ TEZ-1758. TezClient should provide YARN diagnostics when the AM crashes
TEZ-1742. Improve response time of internal preemption
TEZ-1745. TestATSHistoryLoggingService::testATSHistoryLoggingServiceShutdown can be flaky.
TEZ-1747. Increase test timeout for TestSecureShuffle.
http://git-wip-us.apache.org/repos/asf/tez/blob/a2d5768b/tez-api/src/main/java/org/apache/tez/client/TezClient.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClient.java b/tez-api/src/main/java/org/apache/tez/client/TezClient.java
index 049fee3..fc70b48 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClient.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClient.java
@@ -84,12 +84,16 @@ import com.google.protobuf.ServiceException;
public class TezClient {
private static final Log LOG = LogFactory.getLog(TezClient.class);
+
+ @VisibleForTesting
+ static final String NO_CLUSTER_DIAGNOSTICS_MSG = "No cluster diagnostics found.";
private final String clientName;
private ApplicationId sessionAppId;
private ApplicationId lastSubmittedAppId;
private AMConfiguration amConfig;
private FrameworkClient frameworkClient;
+ private String diagnostics;
private boolean isSession;
private boolean sessionStarted = false;
private boolean sessionStopped = false;
@@ -510,9 +514,14 @@ public class TezClient {
case ACCEPTED:
case SUBMITTED:
return TezAppMasterStatus.INITIALIZING;
- case FINISHED:
case FAILED:
case KILLED:
+ diagnostics = appReport.getDiagnostics();
+ LOG.info("App did not succeed. Diagnostics: "
+ + (appReport.getDiagnostics() != null ? appReport.getDiagnostics()
+ : NO_CLUSTER_DIAGNOSTICS_MSG));
+ return TezAppMasterStatus.SHUTDOWN;
+ case FINISHED:
return TezAppMasterStatus.SHUTDOWN;
case RUNNING:
if (!isSession) {
@@ -600,7 +609,8 @@ public class TezClient {
while (true) {
TezAppMasterStatus status = getAppMasterStatus();
if (status.equals(TezAppMasterStatus.SHUTDOWN)) {
- throw new SessionNotRunning("TezSession has already shutdown");
+ throw new SessionNotRunning("TezSession has already shutdown. "
+ + ((diagnostics != null) ? diagnostics : NO_CLUSTER_DIAGNOSTICS_MSG));
}
if (status.equals(TezAppMasterStatus.READY)) {
return;
@@ -647,7 +657,7 @@ public class TezClient {
if (!sessionStarted) {
throw new SessionNotRunning("Session not started");
} else if (sessionStopped) {
- throw new SessionNotRunning("Session stopped");
+ throw new SessionNotRunning("Session stopped by user");
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/a2d5768b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index 7be8013..de4bdd0 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -779,12 +779,17 @@ public class TezClientUtils {
if (appState == YarnApplicationState.FINISHED
|| appState == YarnApplicationState.KILLED
|| appState == YarnApplicationState.FAILED) {
- throw new SessionNotRunning("Application not running"
+ String msg = "Application not running"
+ ", applicationId=" + applicationId
+ ", yarnApplicationState=" + appReport.getYarnApplicationState()
+ ", finalApplicationStatus="
+ appReport.getFinalApplicationStatus()
- + ", trackingUrl=" + appReport.getTrackingUrl());
+ + ", trackingUrl=" + appReport.getTrackingUrl()
+ + ", diagnostics="
+ + (appReport.getDiagnostics() != null ? appReport.getDiagnostics()
+ : TezClient.NO_CLUSTER_DIAGNOSTICS_MSG);
+ LOG.info(msg);
+ throw new SessionNotRunning(msg);
}
return null;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/a2d5768b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientImpl.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientImpl.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientImpl.java
index 0c8ef1a..87e64cd 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientImpl.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientImpl.java
@@ -441,18 +441,6 @@ public class DAGClientImpl extends DAGClient {
}
}
- private ApplicationReport getApplicationReport() {
- ApplicationReport appReport = null;
- try {
- appReport = frameworkClient.getApplicationReport(appId);
- } catch (YarnException e) {
- // do nothing
- } catch (IOException e) {
- // do nothing
- }
- return appReport;
- }
-
private void switchToTimelineClient() throws IOException, TezException {
realClient.close();
realClient = new DAGClientTimelineImpl(appId, dagId, conf, frameworkClient);
http://git-wip-us.apache.org/repos/asf/tez/blob/a2d5768b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
index 0ce6dff..f4d4112 100644
--- a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
+++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.tez.dag.api.DAG;
import org.apache.tez.dag.api.PreWarmVertex;
import org.apache.tez.dag.api.ProcessorDescriptor;
+import org.apache.tez.dag.api.SessionNotRunning;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezConstants;
import org.apache.tez.dag.api.TezException;
@@ -74,6 +75,7 @@ public class TestTezClient {
DAGClientAMProtocolBlockingPB sessionAmProxy;
YarnClient mockYarnClient;
ApplicationId mockAppId;
+ boolean callRealGetSessionAMProxy;
public TezClientForTest(String name, TezConfiguration tezConf,
@Nullable Map<String, LocalResource> localResources,
@@ -89,7 +91,10 @@ public class TestTezClient {
@Override
protected DAGClientAMProtocolBlockingPB getSessionAMProxy(ApplicationId appId)
throws TezException, IOException {
- return sessionAmProxy;
+ if (!callRealGetSessionAMProxy) {
+ return sessionAmProxy;
+ }
+ return super.getSessionAMProxy(appId);
}
}
@@ -317,5 +322,63 @@ public class TestTezClient {
Assert.assertThat(exceptionReference.get(),CoreMatchers. instanceOf(InterruptedException.class));
client.stop();
}
+
+ @Test(timeout = 5000)
+ public void testWaitTillReadyAppFailed() throws Exception {
+ final TezClientForTest client = configure();
+ client.start();
+ String msg = "Application Test Failed";
+ when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState())
+ .thenReturn(YarnApplicationState.NEW).thenReturn(YarnApplicationState.FAILED);
+ when(client.mockYarnClient.getApplicationReport(client.mockAppId).getDiagnostics()).thenReturn(
+ msg);
+ try {
+ client.waitTillReady();
+ Assert.fail();
+ } catch (SessionNotRunning e) {
+ Assert.assertTrue(e.getMessage().contains(msg));
+ }
+ client.stop();
+ }
+
+ @Test(timeout = 5000)
+ public void testWaitTillReadyAppFailedNoDiagnostics() throws Exception {
+ final TezClientForTest client = configure();
+ client.start();
+ when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState())
+ .thenReturn(YarnApplicationState.NEW).thenReturn(YarnApplicationState.FAILED);
+ try {
+ client.waitTillReady();
+ Assert.fail();
+ } catch (SessionNotRunning e) {
+ Assert.assertTrue(e.getMessage().contains(TezClient.NO_CLUSTER_DIAGNOSTICS_MSG));
+ }
+ client.stop();
+ }
+
+ @Test(timeout = 5000)
+ public void testSubmitDAGAppFailed() throws Exception {
+ final TezClientForTest client = configure();
+ client.start();
+
+ client.callRealGetSessionAMProxy = true;
+ String msg = "Application Test Failed";
+ when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState())
+ .thenReturn(YarnApplicationState.KILLED);
+ when(client.mockYarnClient.getApplicationReport(client.mockAppId).getDiagnostics()).thenReturn(
+ msg);
+
+ Vertex vertex = Vertex.create("Vertex", ProcessorDescriptor.create("P"), 1,
+ Resource.newInstance(1, 1));
+ DAG dag = DAG.create("DAG").addVertex(vertex);
+
+ try {
+ client.submitDAG(dag);
+ Assert.fail();
+ } catch (SessionNotRunning e) {
+ Assert.assertTrue(e.getMessage().contains(msg));
+ }
+ client.stop();
+ }
}
[21/50] [abbrv] tez git commit: TEZ-1547. Make use of state change
notifier in VertexManagerPlugins and fix TEZ-1494 without latency penalty
(bikas)
Posted by je...@apache.org.
TEZ-1547. Make use of state change notifier in VertexManagerPlugins and fix TEZ-1494 without latency penalty (bikas)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/cd0ed751
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/cd0ed751
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/cd0ed751
Branch: refs/heads/TEZ-8
Commit: cd0ed751ae502492733738f2d7d7d0ae8e4224e9
Parents: dfef97f
Author: Bikas Saha <bi...@apache.org>
Authored: Tue Nov 4 13:07:06 2014 -0800
Committer: Bikas Saha <bi...@apache.org>
Committed: Tue Nov 4 13:07:06 2014 -0800
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../apache/tez/dag/api/VertexManagerPlugin.java | 21 ++
.../tez/dag/api/VertexManagerPluginContext.java | 50 ++++
.../apache/tez/dag/api/event/VertexState.java | 17 +-
.../tez/runtime/api/InputInitializer.java | 4 +-
.../java/org/apache/tez/dag/app/dag/Vertex.java | 5 +-
.../event/VertexEventManagerUserCodeError.java | 36 +++
.../tez/dag/app/dag/event/VertexEventType.java | 1 +
.../dag/impl/ImmediateStartVertexManager.java | 93 ++++---
.../apache/tez/dag/app/dag/impl/VertexImpl.java | 180 ++++++++++--
.../tez/dag/app/dag/impl/VertexManager.java | 150 ++++++++--
.../tez/dag/app/dag/impl/TestDAGImpl.java | 7 -
.../impl/TestImmediateStartVertexManager.java | 131 +++++++++
.../tez/dag/app/dag/impl/TestVertexImpl.java | 279 ++++++++++++++-----
.../tez/dag/app/dag/impl/TestVertexManager.java | 5 +-
.../dag/app/dag/impl/TestVertexRecovery.java | 48 ++++
.../tez/examples/SortMergeJoinExample.java | 16 +-
.../vertexmanager/ShuffleVertexManager.java | 60 +++-
.../vertexmanager/TestShuffleVertexManager.java | 122 +++++---
.../org/apache/tez/test/TestAMRecovery.java | 5 +-
20 files changed, 1005 insertions(+), 227 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/cd0ed751/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index a321acf..adb4352 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -100,6 +100,8 @@ ALL CHANGES:
TEZ-1731. OnDiskMerger can end up clobbering files across tasks with LocalDiskFetch enabled.
TEZ-1735. Allow setting basic info per DAG for Tez UI.
TEZ-1728. Remove local host name from Fetcher thread name.
+ TEZ-1547. Make use of state change notifier in VertexManagerPlugins and fix
+ TEZ-1494 without latency penalty
Release 0.5.1: 2014-10-02
http://git-wip-us.apache.org/repos/asf/tez/blob/cd0ed751/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPlugin.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPlugin.java b/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPlugin.java
index b494b12..6aa18d6 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPlugin.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPlugin.java
@@ -23,6 +23,7 @@ import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.tez.dag.api.event.VertexStateUpdate;
import org.apache.tez.runtime.api.Event;
import org.apache.tez.runtime.api.events.VertexManagerEvent;
@@ -99,4 +100,24 @@ public abstract class VertexManagerPlugin {
public final VertexManagerPluginContext getContext() {
return this.context;
}
+
+ /**
+ * Receive notifications on vertex state changes.
+ * <p/>
+ * State changes will be received based on the registration via
+ * {@link VertexManagerPluginContext#registerForVertexStateUpdates(String, java.util.Set)}
+ * . Notifications will be received for all registered state changes, and not
+ * just for the latest state update. They will be in order in which the state
+ * change occurred.
+ * </p><br>This method may be invoked concurrently with {@link #onVertexStarted(Map)} etc. and
+ * multi-threading/concurrency implications must be considered.
+ *
+ * @param stateUpdate
+ * an event indicating the name of the vertex, and it's updated
+ * state. Additional information may be available for specific
+ * events, Look at the type hierarchy for
+ * {@link org.apache.tez.dag.api.event.VertexStateUpdate}
+ */
+ public void onVertexStateUpdated(VertexStateUpdate stateUpdate) throws Exception {
+ }
}
http://git-wip-us.apache.org/repos/asf/tez/blob/cd0ed751/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java b/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java
index c1f4bcd..dfa9287 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.tez.dag.api.TaskLocationHint;
+import org.apache.tez.dag.api.event.VertexState;
import org.apache.tez.runtime.api.InputSpecUpdate;
import org.apache.tez.runtime.api.events.InputDataInformationEvent;
@@ -169,4 +170,53 @@ public interface VertexManagerPluginContext {
* @return DAG Attempt number
*/
public int getDAGAttemptNumber();
+
+ /**
+ * Register to get notifications on updates to the specified vertex. Notifications will be sent
+ * via {@link VertexManagerPlugin#onVertexStateUpdated(org.apache.tez.dag.api.event.VertexStateUpdate)}
+ *
+ * This method can only be invoked once. Duplicate invocations will result in an error.
+ *
+ * @param vertexName the vertex name for which notifications are required.
+ * @param stateSet the set of states for which notifications are required. null implies all
+ */
+ void registerForVertexStateUpdates(String vertexName, @Nullable Set<VertexState> stateSet);
+
+ /**
+ * Optional API. No need to call this when the vertex is not fully defined to
+ * start with. E.g. vertex parallelism is not defined, or edges are not
+ * configured. In that case, Tez will assume that the vertex needs
+ * reconfiguration. If the vertex is already fully defined, but the
+ * {@link VertexManagerPlugin} wants to reconfigure the vertex, then it must
+ * use this API to inform Tez about its intention. Without invoking this
+ * method, it is invalid to re-configure the vertex, e.g. via the
+ * {@link #setVertexParallelism(int, VertexLocationHint, Map, Map)} method if
+ * the vertex is already fully defined. This can be invoked at any time until
+ * {@link VertexManagerPlugin#initialize()} has completed. Its invalid to
+ * invoke this method after {@link VertexManagerPlugin#initialize()} has
+ * completed<br>
+ * If this API is invoked, then {@link #doneReconfiguringVertex()} must be
+ * invoked after the {@link VertexManagerPlugin} is done reconfiguring the
+ * vertex, . Actions like scheduling tasks or sending events do not count as
+ * reconfiguration.
+ */
+ public void vertexReconfigurationPlanned();
+
+ /**
+ * Optional API. This needs to be called only if {@link #vertexReconfigurationPlanned()} has been
+ * invoked. This must be called after {@link #vertexReconfigurationPlanned()} is called.
+ */
+ public void doneReconfiguringVertex();
+
+ /**
+ * Optional API. This API can be invoked to declare that the
+ * {@link VertexManagerPlugin} is done with its work. After this the system
+ * will not invoke the plugin methods any more. Its invalid for the plugin to
+ * make further invocations of the context APIs after this. This can be used
+ * to stop receiving further {@link VertexState} notifications after the
+ * plugin has made all changes.
+ */
+ // TODO must be done later after TEZ-1714
+ //public void vertexManagerDone();
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/cd0ed751/tez-api/src/main/java/org/apache/tez/dag/api/event/VertexState.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/event/VertexState.java b/tez-api/src/main/java/org/apache/tez/dag/api/event/VertexState.java
index ab296a5..c9c2d58 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/event/VertexState.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/event/VertexState.java
@@ -28,12 +28,14 @@ import org.apache.hadoop.classification.InterfaceStability;
@InterfaceStability.Unstable
public enum VertexState {
/**
- * Indicates that the Vertex had entered the SUCCEEDED state. A vertex could go back into RUNNING state after SUCCEEDING
+ * Indicates that the Vertex had entered the SUCCEEDED state. A vertex could
+ * go back into RUNNING state after SUCCEEDING
*/
SUCCEEDED,
/**
- * Indicates that the Vertex had entered the RUNNING state. This state can be reached after SUCCEEDED, if some
- * tasks belonging to the vertex are restarted due to errors
+ * Indicates that the Vertex had entered the RUNNING state. This state can be
+ * reached after SUCCEEDED, if some tasks belonging to the vertex are
+ * restarted due to errors
*/
RUNNING,
/**
@@ -47,5 +49,12 @@ public enum VertexState {
/**
* Indicates that the parallelism for the vertex had changed.
*/
- PARALLELISM_UPDATED
+ PARALLELISM_UPDATED,
+ /**
+ * Indicates that the vertex has been completely configured. Parallelism, edges, edge
+ * properties, inputs/outputs have been set and will not be changed any
+ * further. Listeners can depend on the vertex's configured state after
+ * receiving this notification.
+ */
+ CONFIGURED
}
http://git-wip-us.apache.org/repos/asf/tez/blob/cd0ed751/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializer.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializer.java b/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializer.java
index d9d6517..cc33205 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializer.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializer.java
@@ -19,6 +19,7 @@
package org.apache.tez.runtime.api;
import java.util.List;
+import java.util.Map;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.classification.InterfaceAudience.Public;
@@ -97,7 +98,8 @@ public abstract class InputInitializer {
*
* Extensive processing should not be performed via this method call. Instead this should just be
* used as a notification mechanism to the main initialization, which is via the initialize method.
- *
+ * <br>This method may be invoked concurrently with {@link #initialize()} etc. and
+ * multi-threading/concurrency implications must be considered.
* @param stateUpdate an event indicating the name of the vertex, and it's updated state.
* Additional information may be available for specific events, Look at the
* type hierarchy for {@link org.apache.tez.dag.api.event.VertexStateUpdate}
http://git-wip-us.apache.org/repos/asf/tez/blob/cd0ed751/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
index fa1f2c4..cfedc41 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
@@ -89,8 +89,11 @@ public interface Vertex extends Comparable<Vertex> {
void setParallelism(int parallelism, VertexLocationHint vertexLocationHint,
Map<String, EdgeManagerPluginDescriptor> sourceEdgeManagers,
- Map<String, InputSpecUpdate> rootInputSpecUpdate) throws AMUserCodeException;
+ Map<String, InputSpecUpdate> rootInputSpecUpdate, boolean fromVertexManager)
+ throws AMUserCodeException;
void setVertexLocationHint(VertexLocationHint vertexLocationHint);
+ void vertexReconfigurationPlanned();
+ void doneReconfiguringVertex();
// CHANGE THESE TO LISTS AND MAINTAIN ORDER?
void setInputVertices(Map<Vertex, Edge> inVertices);
http://git-wip-us.apache.org/repos/asf/tez/blob/cd0ed751/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventManagerUserCodeError.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventManagerUserCodeError.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventManagerUserCodeError.java
new file mode 100644
index 0000000..022620a
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventManagerUserCodeError.java
@@ -0,0 +1,36 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.tez.dag.app.dag.event;
+
+import org.apache.tez.dag.app.dag.impl.AMUserCodeException;
+import org.apache.tez.dag.records.TezVertexID;
+
+public class VertexEventManagerUserCodeError extends VertexEvent {
+ final AMUserCodeException e;
+
+ public VertexEventManagerUserCodeError(TezVertexID vertexId, AMUserCodeException e) {
+ super(vertexId, VertexEventType.V_MANAGER_USER_CODE_ERROR);
+ this.e = e;
+ }
+
+ public AMUserCodeException getError() {
+ return e;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/cd0ed751/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventType.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventType.java
index e649095..b4f7e29 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventType.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventType.java
@@ -42,6 +42,7 @@ public enum VertexEventType {
//Producer:Any component
V_INTERNAL_ERROR,
+ V_MANAGER_USER_CODE_ERROR,
V_ROUTE_EVENT,
V_ONE_TO_ONE_SOURCE_SPLIT,
http://git-wip-us.apache.org/repos/asf/tez/blob/cd0ed751/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ImmediateStartVertexManager.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ImmediateStartVertexManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ImmediateStartVertexManager.java
index 773426b..00b5306 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ImmediateStartVertexManager.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ImmediateStartVertexManager.java
@@ -18,8 +18,10 @@
package org.apache.tez.dag.app.dag.impl;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.tez.dag.api.EdgeProperty;
@@ -27,11 +29,15 @@ import org.apache.tez.dag.api.InputDescriptor;
import org.apache.tez.dag.api.VertexManagerPlugin;
import org.apache.tez.dag.api.VertexManagerPluginContext;
import org.apache.tez.dag.api.VertexManagerPluginContext.TaskWithLocationHint;
+import org.apache.tez.dag.api.event.VertexState;
+import org.apache.tez.dag.api.event.VertexStateUpdate;
import org.apache.tez.runtime.api.Event;
import org.apache.tez.runtime.api.events.VertexManagerEvent;
+import java.util.EnumSet;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
/**
* Starts all tasks immediately on vertex start
@@ -40,18 +46,10 @@ public class ImmediateStartVertexManager extends VertexManagerPlugin {
private static final Log LOG = LogFactory.getLog(ImmediateStartVertexManager.class);
- private final Map<String, SourceVertexInfo> srcVertexInfo = Maps.newHashMap();
+ private final Map<String, Boolean> srcVertexConfigured = Maps.newConcurrentMap();
private int managedTasks;
private boolean tasksScheduled = false;
-
- class SourceVertexInfo {
- EdgeProperty edgeProperty;
- int numFinishedTasks;
-
- SourceVertexInfo(EdgeProperty edgeProperty) {
- this.edgeProperty = edgeProperty;
- }
- }
+ private AtomicBoolean onVertexStartedDone = new AtomicBoolean(false);
public ImmediateStartVertexManager(VertexManagerPluginContext context) {
super(context);
@@ -63,37 +61,35 @@ public class ImmediateStartVertexManager extends VertexManagerPlugin {
Map<String, EdgeProperty> edges = getContext().getInputVertexEdgeProperties();
for (Map.Entry<String, EdgeProperty> entry : edges.entrySet()) {
String srcVertex = entry.getKey();
- EdgeProperty edgeProp = entry.getValue();
- LOG.info("Task count in " + srcVertex + ": " + getContext().getVertexNumTasks(srcVertex));
//track vertices with task count > 0
if (getContext().getVertexNumTasks(srcVertex) > 0) {
- srcVertexInfo.put(srcVertex, new SourceVertexInfo(edgeProp));
+ LOG.info("Task count in " + srcVertex + ": " + getContext().getVertexNumTasks(srcVertex));
+ srcVertexConfigured.put(srcVertex, false);
+ getContext().registerForVertexStateUpdates(srcVertex, EnumSet.of(VertexState.CONFIGURED));
} else {
LOG.info("Vertex: " + getContext().getVertexName() + "; Ignoring " + srcVertex
+ " as it has got 0 tasks");
}
}
-
- //handle completions
- for (Map.Entry<String, List<Integer>> entry : completions.entrySet()) {
- for (Integer task : entry.getValue()) {
- handleSourceTaskFinished(entry.getKey(), task);
- }
- }
+ onVertexStartedDone.set(true);
scheduleTasks();
}
- private void handleSourceTaskFinished(String vertex, Integer taskId) {
- SourceVertexInfo srcInfo = srcVertexInfo.get(vertex);
- //Not mandatory to check for duplicate completions here
- srcInfo.numFinishedTasks++;
- }
-
private void scheduleTasks() {
- if (!canScheduleTasks()) {
+ if (!onVertexStartedDone.get()) {
+ // vertex not started yet
+ return;
+ }
+ if (tasksScheduled) {
+ // already scheduled
return;
}
+ if (!canScheduleTasks()) {
+ return;
+ }
+
+ tasksScheduled = true;
List<TaskWithLocationHint> tasksToStart = Lists.newArrayListWithCapacity(managedTasks);
for (int i = 0; i < managedTasks; ++i) {
tasksToStart.add(new TaskWithLocationHint(new Integer(i), null));
@@ -103,35 +99,42 @@ public class ImmediateStartVertexManager extends VertexManagerPlugin {
LOG.info("Starting " + tasksToStart.size() + " in " + getContext().getVertexName());
getContext().scheduleVertexTasks(tasksToStart);
}
- tasksScheduled = true;
+ // all tasks scheduled. Can call vertexManagerDone().
+ // TODO TEZ-1714 for locking issues getContext().vertexManagerDone();
}
private boolean canScheduleTasks() {
- //Check if at least 1 task is finished from each source vertex (in case of broadcast &
- // one-to-one or custom)
- for (Map.Entry<String, SourceVertexInfo> entry : srcVertexInfo.entrySet()) {
- SourceVertexInfo srcVertexInfo = entry.getValue();
- switch(srcVertexInfo.edgeProperty.getDataMovementType()) {
- case ONE_TO_ONE:
- case BROADCAST:
- case CUSTOM:
- if (srcVertexInfo.numFinishedTasks == 0) {
- //do not schedule tasks until a task from source task is complete
- return false;
+ // check for source vertices completely configured
+ for (Map.Entry<String, Boolean> entry : srcVertexConfigured.entrySet()) {
+ if (!entry.getValue().booleanValue()) {
+ // vertex not configured
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Waiting for vertex: " + entry.getKey() + " in vertex: " + getContext().getVertexName());
}
- default:
- break;
+ return false;
}
}
+
return true;
}
+
+ @Override
+ public void onVertexStateUpdated(VertexStateUpdate stateUpdate) {
+ Preconditions.checkArgument(stateUpdate.getVertexState() == VertexState.CONFIGURED,
+ "Received incorrect state notification : " + stateUpdate.getVertexState() + " for vertex: "
+ + stateUpdate.getVertexName() + " in vertex: " + getContext().getVertexName());
+ Preconditions.checkArgument(srcVertexConfigured.containsKey(stateUpdate.getVertexName()),
+ "Received incorrect vertex notification : " + stateUpdate.getVertexState() + " for vertex: "
+ + stateUpdate.getVertexName() + " in vertex: " + getContext().getVertexName());
+ Preconditions.checkState(srcVertexConfigured.put(stateUpdate.getVertexName(), true)
+ .booleanValue() == false);
+ LOG.info("Received configured notification: " + stateUpdate.getVertexState() + " for vertex: "
+ + stateUpdate.getVertexName() + " in vertex: " + getContext().getVertexName());
+ scheduleTasks();
+ }
@Override
public void onSourceTaskCompleted(String srcVertexName, Integer attemptId) {
- handleSourceTaskFinished(srcVertexName, attemptId);
- if (!tasksScheduled) {
- scheduleTasks();
- }
}
@Override
http://git-wip-us.apache.org/repos/asf/tez/blob/cd0ed751/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index 4a88949..593ecca 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -110,6 +110,7 @@ import org.apache.tez.dag.app.dag.event.TaskEventRecoverTask;
import org.apache.tez.dag.app.dag.event.TaskEventTermination;
import org.apache.tez.dag.app.dag.event.TaskEventType;
import org.apache.tez.dag.app.dag.event.VertexEvent;
+import org.apache.tez.dag.app.dag.event.VertexEventManagerUserCodeError;
import org.apache.tez.dag.app.dag.event.VertexEventNullEdgeInitialized;
import org.apache.tez.dag.app.dag.event.VertexEventOneToOneSourceSplit;
import org.apache.tez.dag.app.dag.event.VertexEventRecoverVertex;
@@ -320,7 +321,11 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
(VertexState.RECOVERING, VertexState.RECOVERING,
VertexEventType.V_TERMINATE,
new TerminateDuringRecoverTransition())
-
+ .addTransition
+ (VertexState.RECOVERING, EnumSet.of(VertexState.RECOVERING),
+ VertexEventType.V_MANAGER_USER_CODE_ERROR,
+ new VertexManagerUserCodeErrorTransition())
+
// Transitions from INITIALIZING state
.addTransition(VertexState.INITIALIZING,
EnumSet.of(VertexState.INITIALIZING, VertexState.INITED,
@@ -353,6 +358,9 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
EnumSet.of(VertexState.INITIALIZING, VertexState.FAILED),
VertexEventType.V_ROUTE_EVENT,
ROUTE_EVENT_TRANSITION)
+ .addTransition(VertexState.INITIALIZING, EnumSet.of(VertexState.FAILED),
+ VertexEventType.V_MANAGER_USER_CODE_ERROR,
+ new VertexManagerUserCodeErrorTransition())
.addTransition(VertexState.INITIALIZING, VertexState.KILLED,
VertexEventType.V_TERMINATE,
new TerminateInitingVertexTransition())
@@ -399,6 +407,9 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
.addTransition(VertexState.INITED, VertexState.KILLED,
VertexEventType.V_TERMINATE,
new TerminateInitedVertexTransition())
+ .addTransition(VertexState.INITED, EnumSet.of(VertexState.FAILED),
+ VertexEventType.V_MANAGER_USER_CODE_ERROR,
+ new VertexManagerUserCodeErrorTransition())
.addTransition(VertexState.INITED, VertexState.ERROR,
VertexEventType.V_INTERNAL_ERROR,
INTERNAL_ERROR_TRANSITION)
@@ -429,6 +440,9 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
.addTransition(VertexState.RUNNING, VertexState.TERMINATING,
VertexEventType.V_TERMINATE,
new VertexKilledTransition())
+ .addTransition(VertexState.RUNNING, EnumSet.of(VertexState.TERMINATING),
+ VertexEventType.V_MANAGER_USER_CODE_ERROR,
+ new VertexManagerUserCodeErrorTransition())
.addTransition(VertexState.RUNNING, VertexState.RUNNING,
VertexEventType.V_TASK_RESCHEDULED,
new TaskRescheduledTransition())
@@ -460,6 +474,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
// Ignore-able events
.addTransition(VertexState.TERMINATING, VertexState.TERMINATING,
EnumSet.of(VertexEventType.V_TERMINATE,
+ VertexEventType.V_MANAGER_USER_CODE_ERROR,
VertexEventType.V_ROOT_INPUT_FAILED,
VertexEventType.V_SOURCE_VERTEX_STARTED,
VertexEventType.V_ROOT_INPUT_INITIALIZED,
@@ -512,6 +527,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
// Ignore-able events
.addTransition(VertexState.FAILED, VertexState.FAILED,
EnumSet.of(VertexEventType.V_TERMINATE,
+ VertexEventType.V_MANAGER_USER_CODE_ERROR,
VertexEventType.V_ROOT_INPUT_FAILED,
VertexEventType.V_SOURCE_VERTEX_STARTED,
VertexEventType.V_TASK_RESCHEDULED,
@@ -534,6 +550,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
// Ignore-able events
.addTransition(VertexState.KILLED, VertexState.KILLED,
EnumSet.of(VertexEventType.V_TERMINATE,
+ VertexEventType.V_MANAGER_USER_CODE_ERROR,
VertexEventType.V_ROOT_INPUT_FAILED,
VertexEventType.V_INIT,
VertexEventType.V_SOURCE_VERTEX_STARTED,
@@ -559,6 +576,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
VertexEventType.V_START,
VertexEventType.V_ROUTE_EVENT,
VertexEventType.V_TERMINATE,
+ VertexEventType.V_MANAGER_USER_CODE_ERROR,
VertexEventType.V_TASK_COMPLETED,
VertexEventType.V_TASK_ATTEMPT_COMPLETED,
VertexEventType.V_ONE_TO_ONE_SOURCE_SPLIT,
@@ -627,6 +645,10 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
private final String vertexName;
private final ProcessorDescriptor processorDescriptor;
+
+ private boolean vertexToBeReconfiguredByManager = false;
+ AtomicBoolean vmIsInitialized = new AtomicBoolean(false);
+ AtomicBoolean completelyConfiguredSent = new AtomicBoolean(false);
@VisibleForTesting
Map<Vertex, Edge> sourceVertices;
@@ -1212,14 +1234,16 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
@Override
public void setParallelism(int parallelism, VertexLocationHint vertexLocationHint,
Map<String, EdgeManagerPluginDescriptor> sourceEdgeManagers,
- Map<String, InputSpecUpdate> rootInputSpecUpdates) throws AMUserCodeException {
- setParallelism(parallelism, vertexLocationHint, sourceEdgeManagers, rootInputSpecUpdates, false);
+ Map<String, InputSpecUpdate> rootInputSpecUpdates, boolean fromVertexManager)
+ throws AMUserCodeException {
+ setParallelism(parallelism, vertexLocationHint, sourceEdgeManagers, rootInputSpecUpdates,
+ false, fromVertexManager);
}
private void setParallelism(int parallelism, VertexLocationHint vertexLocationHint,
Map<String, EdgeManagerPluginDescriptor> sourceEdgeManagers,
Map<String, InputSpecUpdate> rootInputSpecUpdates,
- boolean recovering) throws AMUserCodeException {
+ boolean recovering, boolean fromVertexManager) throws AMUserCodeException {
if (recovering) {
writeLock.lock();
try {
@@ -1255,6 +1279,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
+ parallelism + " for vertex: " + logIdentifier);
setVertexLocationHint(vertexLocationHint);
writeLock.lock();
+
try {
if (parallelismSet == true) {
String msg = "Parallelism can only be set dynamically once per vertex: " + logIdentifier;
@@ -1262,6 +1287,15 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
throw new TezUncheckedException(msg);
}
+ if (fromVertexManager && canInitVertex()) {
+ // vertex is fully defined. setParallelism has been called. VertexManager should have
+ // informed us about this. Otherwise we would have notified listeners that we are fully
+ // defined before we are actually fully defined
+ Preconditions.checkState(vertexToBeReconfiguredByManager, "Vertex is fully configured but still"
+ + " the reconfiguration API has been called. VertexManager must notify the framework using "
+ + " context.vertexReconfigurationPlanned() before re-configuring the vertex.");
+ }
+
parallelismSet = true;
// Input initializer/Vertex Manager/1-1 split expected to set parallelism.
@@ -1421,6 +1455,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
}
}
+ @Override
public void setVertexLocationHint(VertexLocationHint vertexLocationHint) {
writeLock.lock();
try {
@@ -1432,6 +1467,46 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
writeLock.unlock();
}
}
+
+ @Override
+ public void vertexReconfigurationPlanned() {
+ vertexReconfigurationPlanned(false);
+ }
+
+ public void vertexReconfigurationPlanned(boolean testOverride) {
+ writeLock.lock();
+ try {
+ if (testOverride) {
+ Preconditions.checkState(vmIsInitialized.get() && completelyConfiguredSent.get(),
+ "test should override only failed cases");
+ } else {
+ Preconditions.checkState(!vmIsInitialized.get(),
+ "context.vertexReconfigurationPlanned() cannot be called after initialize()");
+ Preconditions.checkState(!completelyConfiguredSent.get(), "vertexReconfigurationPlanned() "
+ + " cannot be invoked after the vertex has been configured.");
+ }
+ this.vertexToBeReconfiguredByManager = true;
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public void doneReconfiguringVertex() {
+ writeLock.lock();
+ try {
+ Preconditions.checkState(vertexToBeReconfiguredByManager, "doneReconfiguringVertex() can be "
+ + "invoked only after vertexReconfigurationPlanned() is invoked");
+ this.vertexToBeReconfiguredByManager = false;
+ if (completelyConfiguredSent.compareAndSet(false, true)) {
+ // vertex already started and at that time this event was not sent. Send now.
+ stateChangeNotifier.stateChanged(vertexId, new VertexStateUpdate(vertexName,
+ org.apache.tez.dag.api.event.VertexState.CONFIGURED));
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
@Override
/**
@@ -1970,6 +2045,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
try {
vertexManager.initialize();
+ vmIsInitialized.set(true);
} catch (AMUserCodeException e) {
String msg = "Exception in " + e.getSource()+ ", vertex:" + logIdentifier;
LOG.error(msg, e);
@@ -2042,7 +2118,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
.getVertexManagerPlugin());
LOG.info("Setting user vertex manager plugin: "
+ pluginDesc.getClassName() + " on vertex: " + getName());
- vertexManager = new VertexManager(pluginDesc, this, appContext);
+ vertexManager = new VertexManager(pluginDesc, this, appContext, stateChangeNotifier);
} else {
// Intended order of picking a vertex manager
// If there is an InputInitializer then we use the RootInputVertexManager. May be fixed by TEZ-703
@@ -2055,26 +2131,26 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
+ logIdentifier);
vertexManager = new VertexManager(
VertexManagerPluginDescriptor.create(RootInputVertexManager.class.getName()),
- this, appContext);
+ this, appContext, stateChangeNotifier);
} else if (hasOneToOne && !hasCustom) {
LOG.info("Setting vertexManager to InputReadyVertexManager for "
+ logIdentifier);
vertexManager = new VertexManager(
VertexManagerPluginDescriptor.create(InputReadyVertexManager.class.getName()),
- this, appContext);
+ this, appContext, stateChangeNotifier);
} else if (hasBipartite && !hasCustom) {
LOG.info("Setting vertexManager to ShuffleVertexManager for "
+ logIdentifier);
// shuffle vertex manager needs a conf payload
vertexManager = new VertexManager(ShuffleVertexManager.createConfigBuilder(conf).build(),
- this, appContext);
+ this, appContext, stateChangeNotifier);
} else {
// schedule all tasks upon vertex start. Default behavior.
LOG.info("Setting vertexManager to ImmediateStartVertexManager for "
+ logIdentifier);
vertexManager = new VertexManager(
VertexManagerPluginDescriptor.create(ImmediateStartVertexManager.class.getName()),
- this, appContext);
+ this, appContext, stateChangeNotifier);
}
}
}
@@ -2222,7 +2298,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
new TaskEventRecoverTask(task.getTaskId()));
}
try {
- vertex.vertexManager.onVertexStarted(vertex.pendingReportedSrcCompletions);
+ vertex.recoveryCodeSimulatingStart();
endState = VertexState.RUNNING;
} catch (AMUserCodeException e) {
String msg = "Exception in " + e.getSource() + ", vertex:" + vertex.getLogIdentifier();
@@ -2274,7 +2350,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
taskState));
}
try {
- vertex.vertexManager.onVertexStarted(vertex.pendingReportedSrcCompletions);
+ vertex.recoveryCodeSimulatingStart();
endState = VertexState.RUNNING;
} catch (AMUserCodeException e) {
String msg = "Exception in " + e.getSource() +", vertex:" + vertex.getLogIdentifier();
@@ -2335,6 +2411,13 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
}
}
+
+ private void recoveryCodeSimulatingStart() throws AMUserCodeException {
+ vertexManager.onVertexStarted(pendingReportedSrcCompletions);
+ // This code is duplicated from startVertex() because recovery does not follow normal
+ // transitions. To be removed after recovery code is fixed.
+ maybeSendConfiguredEvent();
+ }
private void routeRecoveredEvents(VertexState vertexState,
List<TezEvent> tezEvents) {
@@ -2556,7 +2639,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
boolean successSetParallelism ;
try {
vertex.setParallelism(0,
- null, vertex.recoveredSourceEdgeManagers, vertex.recoveredRootInputSpecUpdates, true);
+ null, vertex.recoveredSourceEdgeManagers, vertex.recoveredRootInputSpecUpdates, true, false);
successSetParallelism = true;
} catch (Exception e) {
successSetParallelism = false;
@@ -2614,7 +2697,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
}
try {
vertex.setParallelism(0, null, vertex.recoveredSourceEdgeManagers,
- vertex.recoveredRootInputSpecUpdates, true);
+ vertex.recoveredRootInputSpecUpdates, true, false);
successSetParallelism = true;
} catch (Exception e) {
successSetParallelism = false;
@@ -2634,7 +2717,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
new TaskEventRecoverTask(task.getTaskId()));
}
try {
- vertex.vertexManager.onVertexStarted(vertex.pendingReportedSrcCompletions);
+ vertex.recoveryCodeSimulatingStart();
endState = VertexState.RUNNING;
} catch (AMUserCodeException e) {
String msg = "Exception in " + e.getSource() + ", vertex=" + vertex.getLogIdentifier();
@@ -2674,7 +2757,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
}
// Wait for all tasks to recover and report back
try {
- vertex.vertexManager.onVertexStarted(vertex.pendingReportedSrcCompletions);
+ vertex.recoveryCodeSimulatingStart();
endState = VertexState.RUNNING;
} catch (AMUserCodeException e) {
String msg = "Exception in " + e.getSource() +", vertex:" + vertex.getLogIdentifier();
@@ -2825,10 +2908,12 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
// Create tasks based on initial configuration, but don't start them yet.
if (vertex.numTasks == -1) {
+ // this block must always return VertexState.INITIALIZING
LOG.info("Num tasks is -1. Expecting VertexManager/InputInitializers/1-1 split"
+ " to set #tasks for the vertex " + vertex.getVertexId());
if (vertex.inputsWithInitializers != null) {
+ LOG.info("Vertex will initialize from input initializer. " + vertex.logIdentifier);
vertex.setupInputInitializerManager();
return VertexState.INITIALIZING;
} else {
@@ -2857,8 +2942,9 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
} else {
LOG.info("Creating " + vertex.numTasks + " for vertex: " + vertex.logIdentifier);
vertex.createTasks();
-
+ // this block may return VertexState.INITIALIZING
if (vertex.inputsWithInitializers != null) {
+ LOG.info("Vertex will initialize from input initializer. " + vertex.logIdentifier);
vertex.setupInputInitializerManager();
return VertexState.INITIALIZING;
}
@@ -2867,6 +2953,8 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
return VertexState.INITIALIZING;
}
LOG.info("Directly initializing vertex: " + vertex.logIdentifier);
+ // vertex is completely configured. Send out notification now.
+ vertex.maybeSendConfiguredEvent();
boolean isInitialized = vertex.initializeVertex();
if (isInitialized) {
return VertexState.INITED;
@@ -3020,7 +3108,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
" numTasks " + splitEvent.getNumTasks());
vertex.originalOneToOneSplitSource = originalSplitSource;
try {
- vertex.setParallelism(splitEvent.getNumTasks(), null, null, null);
+ vertex.setParallelism(splitEvent.getNumTasks(), null, null, null, false);
} catch (Exception e) {
// ingore this exception, should not happen
LOG.error("Unexpected exception, Just set Parallelims to a specified value, not involve EdgeManager,"
@@ -3130,8 +3218,25 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
return vertex.startVertex();
}
}
+
+ private void maybeSendConfiguredEvent() {
+ // the vertex is fully configured by the time it starts. Always notify completely configured
+ // unless the vertex manager has told us that it is going to reconfigure it further
+ Preconditions.checkState(canInitVertex());
+ if (!this.vertexToBeReconfiguredByManager) {
+ // this vertex will not be reconfigured by its manager
+ if (completelyConfiguredSent.compareAndSet(false, true)) {
+ stateChangeNotifier.stateChanged(vertexId, new VertexStateUpdate(vertexName,
+ org.apache.tez.dag.api.event.VertexState.CONFIGURED));
+ }
+ }
+ }
private VertexState startVertex() {
+ // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ // IMPORTANT - Until Recovery is fixed to use normal state transitions, if any code is added
+ // here then please check if it needs to be duplicated in recoveryCodeSimulatingStart().
+ // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Preconditions.checkState(getState() == VertexState.INITED,
"Vertex must be inited " + logIdentifier);
@@ -3147,7 +3252,16 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
}
pendingReportedSrcCompletions.clear();
logJobHistoryVertexStartedEvent();
-
+
+ // the vertex is fully configured by the time it starts. Always notify completely configured
+ // unless the vertex manager has told us that it is going to reconfigure it further.
+ // If the vertex was pre-configured then the event would have been sent out earlier. Calling again
+ // would be a no-op. If the vertex was not fully configured and waiting for that to complete then
+ // we would start immediately after that. Either parallelism updated (now) or IPO changed (future)
+ // or vertex added (future). Simplify these cases by sending the event now automatically for the
+ // user as if they had invoked the planned()/done() API's.
+ maybeSendConfiguredEvent();
+
// TODO: Metrics
//job.metrics.runningJob(job);
@@ -3309,6 +3423,36 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
}
}
+ private static class VertexManagerUserCodeErrorTransition implements
+ MultipleArcTransition<VertexImpl, VertexEvent, VertexState> {
+ @Override
+ public VertexState transition(VertexImpl vertex, VertexEvent event) {
+ VertexEventManagerUserCodeError errEvent = ((VertexEventManagerUserCodeError) event);
+ AMUserCodeException e = errEvent.getError();
+ String msg = "Exception in " + e.getSource() + ", vertex:" + vertex.getLogIdentifier();
+ LOG.error(msg, e);
+
+ if (vertex.getState() == VertexState.RECOVERING) {
+ LOG.info("Received a user code error during recovering, setting recovered"
+ + " state to FAILED");
+ vertex.addDiagnostic(msg + "," + ExceptionUtils.getStackTrace(e.getCause()));
+ vertex.terminationCause = VertexTerminationCause.AM_USERCODE_FAILURE;
+ vertex.recoveredState = VertexState.FAILED;
+ return VertexState.RECOVERING;
+ } else if (vertex.getState() == VertexState.RUNNING) {
+ vertex.addDiagnostic(msg + "," + ExceptionUtils.getStackTrace(e.getCause()));
+ vertex.tryEnactKill(VertexTerminationCause.AM_USERCODE_FAILURE,
+ TaskTerminationCause.AM_USERCODE_FAILURE);
+ return VertexState.TERMINATING;
+ } else {
+ vertex.finished(VertexState.FAILED,
+ VertexTerminationCause.AM_USERCODE_FAILURE, msg
+ + ", " + ExceptionUtils.getStackTrace(e.getCause()));
+ return VertexState.FAILED;
+ }
+ }
+ }
+
/**
* Here, the Vertex is being told that one of it's source task-attempts
* completed.
http://git-wip-us.apache.org/repos/asf/tez/blob/cd0ed751/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java
index 1bfb0f9..dd38c2a 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java
@@ -27,6 +27,7 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicBoolean;
import javax.annotation.Nullable;
@@ -46,11 +47,16 @@ import org.apache.tez.dag.api.VertexLocationHint;
import org.apache.tez.dag.api.VertexManagerPlugin;
import org.apache.tez.dag.api.VertexManagerPluginContext;
import org.apache.tez.dag.api.VertexManagerPluginDescriptor;
+import org.apache.tez.dag.api.event.VertexState;
+import org.apache.tez.dag.api.event.VertexStateUpdate;
import org.apache.tez.dag.app.AppContext;
+import org.apache.tez.dag.app.dag.StateChangeNotifier;
import org.apache.tez.dag.app.dag.Task;
import org.apache.tez.dag.app.dag.TaskAttempt;
import org.apache.tez.dag.app.dag.Vertex;
+import org.apache.tez.dag.app.dag.event.VertexEventManagerUserCodeError;
import org.apache.tez.dag.app.dag.impl.AMUserCodeException.Source;
+import org.apache.tez.dag.app.dag.VertexStateUpdateListener;
import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.dag.records.TezTaskID;
import org.apache.tez.runtime.api.Event;
@@ -75,18 +81,27 @@ public class VertexManager {
UserPayload payload = null;
AppContext appContext;
BlockingQueue<TezEvent> rootInputInitEventQueue;
+ StateChangeNotifier stateChangeNotifier;
private static final Log LOG = LogFactory.getLog(VertexManager.class);
- class VertexManagerPluginContextImpl implements VertexManagerPluginContext {
- // TODO Add functionality to allow VertexManagers to send VertexManagerEvents
+ class VertexManagerPluginContextImpl implements VertexManagerPluginContext, VertexStateUpdateListener {
private EventMetaData rootEventSourceMetadata = new EventMetaData(EventProducerConsumerType.INPUT,
managedVertex.getName(), "NULL_VERTEX", null);
private Map<String, EventMetaData> destinationEventMetadataMap = Maps.newHashMap();
+ private final List<String> notificationRegisteredVertices = Lists.newArrayList();
+ AtomicBoolean isComplete = new AtomicBoolean(false);
+ private void checkAndThrowIfDone() {
+ if (isComplete()) {
+ throw new TezUncheckedException("Cannot invoke context methods after reporting done");
+ }
+ }
+
@Override
- public Map<String, EdgeProperty> getInputVertexEdgeProperties() {
+ public synchronized Map<String, EdgeProperty> getInputVertexEdgeProperties() {
+ checkAndThrowIfDone();
// TODO Something similar for Initial Inputs - payload etc visible
Map<Vertex, Edge> inputs = managedVertex.getInputVertices();
Map<String, EdgeProperty> vertexEdgeMap =
@@ -98,22 +113,25 @@ public class VertexManager {
}
@Override
- public String getVertexName() {
+ public synchronized String getVertexName() {
+ checkAndThrowIfDone();
return managedVertex.getName();
}
@Override
- public int getVertexNumTasks(String vertexName) {
+ public synchronized int getVertexNumTasks(String vertexName) {
+ checkAndThrowIfDone();
return appContext.getCurrentDAG().getVertex(vertexName).getTotalTasks();
}
@Override
- public void setVertexParallelism(int parallelism, VertexLocationHint vertexLocationHint,
+ public synchronized void setVertexParallelism(int parallelism, VertexLocationHint vertexLocationHint,
Map<String, EdgeManagerPluginDescriptor> sourceEdgeManagers,
Map<String, InputSpecUpdate> rootInputSpecUpdate) {
+ checkAndThrowIfDone();
try {
managedVertex.setParallelism(parallelism, vertexLocationHint, sourceEdgeManagers,
- rootInputSpecUpdate);
+ rootInputSpecUpdate, true);
} catch (AMUserCodeException e) {
// workaround: convert it to TezUncheckedException which would be caught in VM
throw new TezUncheckedException(e);
@@ -121,13 +139,15 @@ public class VertexManager {
}
@Override
- public void scheduleVertexTasks(List<TaskWithLocationHint> tasks) {
+ public synchronized void scheduleVertexTasks(List<TaskWithLocationHint> tasks) {
+ checkAndThrowIfDone();
managedVertex.scheduleTasks(tasks);
}
@Nullable
@Override
- public Set<String> getVertexInputNames() {
+ public synchronized Set<String> getVertexInputNames() {
+ checkAndThrowIfDone();
Set<String> inputNames = null;
Map<String, RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>>
inputs = managedVertex.getAdditionalInputs();
@@ -138,13 +158,15 @@ public class VertexManager {
}
@Override
- public UserPayload getUserPayload() {
+ public synchronized UserPayload getUserPayload() {
+ checkAndThrowIfDone();
return payload;
}
@Override
- public void addRootInputEvents(final String inputName,
+ public synchronized void addRootInputEvents(final String inputName,
Collection<InputDataInformationEvent> events) {
+ checkAndThrowIfDone();
verifyIsRootInput(inputName);
Collection<TezEvent> tezEvents = Collections2.transform(events,
new Function<InputDataInformationEvent, TezEvent>() {
@@ -166,13 +188,15 @@ public class VertexManager {
@Override
- public void setVertexLocationHint(VertexLocationHint locationHint) {
+ public synchronized void setVertexLocationHint(VertexLocationHint locationHint) {
+ checkAndThrowIfDone();
Preconditions.checkNotNull(locationHint, "locationHint is null");
managedVertex.setVertexLocationHint(locationHint);
}
@Override
- public int getDAGAttemptNumber() {
+ public synchronized int getDAGAttemptNumber() {
+ checkAndThrowIfDone();
return appContext.getApplicationAttemptId().getAttemptId();
}
@@ -192,22 +216,26 @@ public class VertexManager {
}
@Override
- public Resource getVertexTaskResource() {
+ public synchronized Resource getVertexTaskResource() {
+ checkAndThrowIfDone();
return managedVertex.getTaskResource();
}
@Override
- public Resource getTotalAvailableResource() {
+ public synchronized Resource getTotalAvailableResource() {
+ checkAndThrowIfDone();
return appContext.getTaskScheduler().getTotalResources();
}
@Override
- public int getNumClusterNodes() {
+ public synchronized int getNumClusterNodes() {
+ checkAndThrowIfDone();
return appContext.getTaskScheduler().getNumClusterNodes();
}
@Override
- public Container getTaskContainer(String vertexName, Integer taskIndex) {
+ public synchronized Container getTaskContainer(String vertexName, Integer taskIndex) {
+ checkAndThrowIfDone();
Vertex vertex = appContext.getCurrentDAG().getVertex(vertexName);
Task task = vertex.getTask(taskIndex.intValue());
TaskAttempt attempt = task.getSuccessfulAttempt();
@@ -216,16 +244,82 @@ public class VertexManager {
}
return null;
}
+
+ @Override
+ public synchronized void registerForVertexStateUpdates(String vertexName, Set<VertexState> stateSet) {
+ checkAndThrowIfDone();
+ synchronized(notificationRegisteredVertices) {
+ notificationRegisteredVertices.add(vertexName);
+ }
+ stateChangeNotifier.registerForVertexUpdates(vertexName, stateSet, this);
+ }
+
+ private void unregisterForVertexStateUpdates() {
+ synchronized (notificationRegisteredVertices) {
+ for (String vertexName : notificationRegisteredVertices) {
+ stateChangeNotifier.unregisterForVertexUpdates(vertexName, this);
+ }
+
+ }
+ }
+
+ boolean isComplete() {
+ return (isComplete.get() == true);
+ }
+
+ // TODO add later after TEZ-1714 @Override
+ public synchronized void vertexManagerDone() {
+ checkAndThrowIfDone();
+ LOG.info("Vertex Manager reported done for : " + managedVertex.getLogIdentifier());
+ this.isComplete.set(true);
+ unregisterForVertexStateUpdates();
+ }
+
+ @Override
+ public synchronized void vertexReconfigurationPlanned() {
+ checkAndThrowIfDone();
+ managedVertex.vertexReconfigurationPlanned();
+ }
+
+ @Override
+ public synchronized void doneReconfiguringVertex() {
+ checkAndThrowIfDone();
+ managedVertex.doneReconfiguringVertex();
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public synchronized void onStateUpdated(VertexStateUpdate event) {
+ if (isComplete()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Dropping state update for vertex=" + event.getVertexName() + ", state=" +
+ event.getVertexState() +
+ " since vertexmanager for " + managedVertex.getLogIdentifier() + " is complete.");
+ }
+ } else {
+ try {
+ plugin.onVertexStateUpdated(event);
+ } catch (Exception e) {
+ // state change must be triggered via an event transition
+ appContext.getEventHandler().handle(
+ new VertexEventManagerUserCodeError(managedVertex.getVertexId(),
+ new AMUserCodeException(Source.VertexManager, e)));
+ }
+ }
+ }
+
}
public VertexManager(VertexManagerPluginDescriptor pluginDesc,
- Vertex managedVertex, AppContext appContext) {
+ Vertex managedVertex, AppContext appContext, StateChangeNotifier stateChangeNotifier) {
checkNotNull(pluginDesc, "pluginDesc is null");
checkNotNull(managedVertex, "managedVertex is null");
checkNotNull(appContext, "appContext is null");
+ checkNotNull(stateChangeNotifier, "notifier is null");
this.pluginDesc = pluginDesc;
this.managedVertex = managedVertex;
this.appContext = appContext;
+ this.stateChangeNotifier = stateChangeNotifier;
// don't specify the size of rootInputInitEventQueue, otherwise it will fail when addAll
this.rootInputInitEventQueue = new LinkedBlockingQueue<TezEvent>();
}
@@ -242,7 +336,9 @@ public class VertexManager {
payload = pluginDesc.getUserPayload();
}
try {
- plugin.initialize();
+ if (!pluginContext.isComplete()) {
+ plugin.initialize();
+ }
} catch (Exception e) {
throw new AMUserCodeException(Source.VertexManager, e);
}
@@ -265,7 +361,9 @@ public class VertexManager {
}
}
try {
- plugin.onVertexStarted(pluginCompletionsMap);
+ if (!pluginContext.isComplete()) {
+ plugin.onVertexStarted(pluginCompletionsMap);
+ }
} catch (Exception e) {
throw new AMUserCodeException(Source.VertexManager, e);
}
@@ -276,7 +374,9 @@ public class VertexManager {
String vertexName =
appContext.getCurrentDAG().getVertex(tezTaskId.getVertexID()).getName();
try {
- plugin.onSourceTaskCompleted(vertexName, taskId);
+ if (!pluginContext.isComplete()) {
+ plugin.onSourceTaskCompleted(vertexName, taskId);
+ }
} catch (Exception e) {
throw new AMUserCodeException(Source.VertexManager, e);
}
@@ -284,7 +384,9 @@ public class VertexManager {
public void onVertexManagerEventReceived(VertexManagerEvent vmEvent) throws AMUserCodeException {
try {
- plugin.onVertexManagerEventReceived(vmEvent);
+ if (!pluginContext.isComplete()) {
+ plugin.onVertexManagerEventReceived(vmEvent);
+ }
} catch (Exception e) {
throw new AMUserCodeException(Source.VertexManager, e);
}
@@ -293,7 +395,9 @@ public class VertexManager {
public List<TezEvent> onRootVertexInitialized(String inputName,
InputDescriptor inputDescriptor, List<Event> events) throws AMUserCodeException {
try {
- plugin.onRootVertexInitialized(inputName, inputDescriptor, events);
+ if (!pluginContext.isComplete()) {
+ plugin.onRootVertexInitialized(inputName, inputDescriptor, events);
+ }
} catch (Exception e) {
throw new AMUserCodeException(Source.VertexManager, e);
}
http://git-wip-us.apache.org/repos/asf/tez/blob/cd0ed751/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
index f1961aa..d859ae0 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
@@ -898,13 +898,10 @@ public class TestDAGImpl {
dispatcher.getEventHandler().handle(new DAGEventStartDag(dagWithCustomEdge.getID(),
null));
dispatcher.await();
- Assert.assertEquals(DAGState.RUNNING, dagWithCustomEdge.getState());
VertexImpl v2 = (VertexImpl)dagWithCustomEdge.getVertex("vertex2");
LOG.info(v2.getTasks().size());
Task t1= v2.getTask(0);
- dispatcher.getEventHandler().handle(new TaskEvent(t1.getTaskId(), TaskEventType.T_SCHEDULE));
- dispatcher.await();
TaskAttemptImpl ta1= (TaskAttemptImpl)t1.getAttempt(TezTaskAttemptID.getInstance(t1.getTaskId(), 0));
Assert.assertEquals(TaskAttemptStateInternal.FAILED, ta1.getInternalState());
@@ -947,7 +944,6 @@ public class TestDAGImpl {
VertexImpl v1 = (VertexImpl)dagWithCustomEdge.getVertex("vertex1");
VertexImpl v2 = (VertexImpl)dagWithCustomEdge.getVertex("vertex2");
- v2.scheduleTasks(Collections.singletonList(new TaskWithLocationHint(new Integer(0), null)));
dispatcher.await();
Task t1= v2.getTask(0);
TaskAttemptImpl ta1= (TaskAttemptImpl)t1.getAttempt(TezTaskAttemptID.getInstance(t1.getTaskId(), 0));
@@ -977,7 +973,6 @@ public class TestDAGImpl {
VertexImpl v1 = (VertexImpl)dagWithCustomEdge.getVertex("vertex1");
VertexImpl v2 = (VertexImpl)dagWithCustomEdge.getVertex("vertex2");
- v2.scheduleTasks(Collections.singletonList(new TaskWithLocationHint(new Integer(0), null)));
dispatcher.await();
Task t1= v2.getTask(0);
@@ -1007,7 +1002,6 @@ public class TestDAGImpl {
VertexImpl v1 = (VertexImpl)dagWithCustomEdge.getVertex("vertex1");
VertexImpl v2 = (VertexImpl)dagWithCustomEdge.getVertex("vertex2");
- v2.scheduleTasks(Collections.singletonList(new TaskWithLocationHint(new Integer(0), null)));
dispatcher.await();
Task t1= v2.getTask(0);
@@ -1038,7 +1032,6 @@ public class TestDAGImpl {
VertexImpl v1 = (VertexImpl)dagWithCustomEdge.getVertex("vertex1");
VertexImpl v2 = (VertexImpl)dagWithCustomEdge.getVertex("vertex2");
- v2.scheduleTasks(Collections.singletonList(new TaskWithLocationHint(new Integer(0), null)));
dispatcher.await();
Task t1= v2.getTask(0);
http://git-wip-us.apache.org/repos/asf/tez/blob/cd0ed751/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestImmediateStartVertexManager.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestImmediateStartVertexManager.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestImmediateStartVertexManager.java
new file mode 100644
index 0000000..6d071a7
--- /dev/null
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestImmediateStartVertexManager.java
@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.dag.impl;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+
+import org.apache.tez.dag.api.EdgeManagerPluginDescriptor;
+import org.apache.tez.dag.api.EdgeProperty;
+import org.apache.tez.dag.api.InputDescriptor;
+import org.apache.tez.dag.api.OutputDescriptor;
+import org.apache.tez.dag.api.VertexManagerPluginContext;
+import org.apache.tez.dag.api.EdgeProperty.SchedulingType;
+import org.apache.tez.dag.api.VertexManagerPluginContext.TaskWithLocationHint;
+import org.apache.tez.dag.api.event.VertexState;
+import org.apache.tez.dag.api.event.VertexStateUpdate;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import static org.mockito.Matchers.anyList;
+import static org.mockito.Mockito.anySet;
+import static org.mockito.Mockito.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class TestImmediateStartVertexManager {
+
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ @Test (timeout=5000)
+ public void testBasic() {
+ HashMap<String, EdgeProperty> mockInputVertices =
+ new HashMap<String, EdgeProperty>();
+ final String mockSrcVertexId1 = "Vertex1";
+ EdgeProperty eProp1 = EdgeProperty.create(
+ EdgeProperty.DataMovementType.SCATTER_GATHER,
+ EdgeProperty.DataSourceType.PERSISTED,
+ SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create("out"),
+ InputDescriptor.create("in"));
+ final String mockSrcVertexId2 = "Vertex2";
+ EdgeProperty eProp2 = EdgeProperty.create(mock(EdgeManagerPluginDescriptor.class),
+ EdgeProperty.DataSourceType.PERSISTED,
+ SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create("out"),
+ InputDescriptor.create("in"));
+ final String mockSrcVertexId3 = "Vertex3";
+ EdgeProperty eProp3 = EdgeProperty.create(
+ EdgeProperty.DataMovementType.BROADCAST,
+ EdgeProperty.DataSourceType.PERSISTED,
+ SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create("out"),
+ InputDescriptor.create("in"));
+
+ final String mockManagedVertexId = "Vertex4";
+
+ mockInputVertices.put(mockSrcVertexId1, eProp1);
+ mockInputVertices.put(mockSrcVertexId2, eProp2);
+ mockInputVertices.put(mockSrcVertexId3, eProp3);
+
+ final VertexManagerPluginContext mockContext = mock(VertexManagerPluginContext.class);
+ when(mockContext.getInputVertexEdgeProperties()).thenReturn(mockInputVertices);
+ when(mockContext.getVertexName()).thenReturn(mockManagedVertexId);
+ when(mockContext.getVertexNumTasks(mockManagedVertexId)).thenReturn(4);
+ when(mockContext.getVertexNumTasks(mockSrcVertexId1)).thenReturn(2);
+ when(mockContext.getVertexNumTasks(mockSrcVertexId2)).thenReturn(2);
+ when(mockContext.getVertexNumTasks(mockSrcVertexId3)).thenReturn(2);
+
+ final HashSet<Integer> scheduledTasks = new HashSet<Integer>();
+ doAnswer(new Answer() {
+ public Object answer(InvocationOnMock invocation) {
+ Object[] args = invocation.getArguments();
+ scheduledTasks.clear();
+ List<TaskWithLocationHint> tasks = (List<TaskWithLocationHint>)args[0];
+ for (TaskWithLocationHint task : tasks) {
+ scheduledTasks.add(task.getTaskIndex());
+ }
+ return null;
+ }}).when(mockContext).scheduleVertexTasks(anyList());
+
+ ImmediateStartVertexManager manager = new ImmediateStartVertexManager(mockContext);
+ manager.initialize();
+ manager.onVertexStarted(null);
+ verify(mockContext, times(0)).scheduleVertexTasks(anyList());
+ manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId1,
+ VertexState.CONFIGURED));
+ manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId2,
+ VertexState.CONFIGURED));
+ manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId3,
+ VertexState.CONFIGURED));
+ verify(mockContext, times(1)).scheduleVertexTasks(anyList());
+ Assert.assertEquals(4, scheduledTasks.size());
+
+ // simulate race between onVertexStarted and notifications
+ scheduledTasks.clear();
+ final ImmediateStartVertexManager raceManager = new ImmediateStartVertexManager(mockContext);
+ doAnswer(new Answer() {
+ public Object answer(InvocationOnMock invocation) throws Exception {
+ raceManager.onVertexStateUpdated(new VertexStateUpdate((String)invocation.getArguments()[0],
+ VertexState.CONFIGURED));
+ scheduledTasks.clear();
+ return null;
+ }}).when(mockContext).registerForVertexStateUpdates(anyString(), anySet());
+ raceManager.initialize();
+ raceManager.onVertexStarted(null);
+ verify(mockContext, times(2)).scheduleVertexTasks(anyList());
+ Assert.assertEquals(4, scheduledTasks.size());
+ }
+
+}
[02/50] [abbrv] tez git commit: TEZ-1689. addendum - fix unit test
failure. (zjffdu)
Posted by je...@apache.org.
TEZ-1689. addendum - fix unit test failure. (zjffdu)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/1ffbc193
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/1ffbc193
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/1ffbc193
Branch: refs/heads/TEZ-8
Commit: 1ffbc1935646f7c422b551e6e0ffdc001311d074
Parents: 9c16607
Author: Jeff Zhang <zj...@apache.org>
Authored: Wed Oct 29 18:58:49 2014 +0800
Committer: Jeff Zhang <zj...@apache.org>
Committed: Wed Oct 29 18:58:49 2014 +0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java | 2 +-
2 files changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/1ffbc193/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 6bf95ae..0d2ccb9 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -74,6 +74,7 @@ ALL CHANGES:
TEZ-1689. Exception handling for EdgeManagerPlugin.
TEZ-1701. ATS fixes to flush all history events and also using batching.
TEZ-792. Default staging path should have user name.
+ TEZ-1689. addendum - fix unit test failure.
Release 0.5.1: 2014-10-02
http://git-wip-us.apache.org/repos/asf/tez/blob/1ffbc193/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
index a597d49..dc90a6f 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
@@ -918,7 +918,7 @@ public class TestDAGImpl {
dispatcher.getEventHandler().handle(new DAGEventStartDag(dagWithCustomEdge.getID(),
null));
dispatcher.await();
- Assert.assertEquals(DAGState.FAILED, dagWithCustomEdge.getState());
+ Assert.assertEquals(DAGState.RUNNING, dagWithCustomEdge.getState());
VertexImpl v1 = (VertexImpl)dagWithCustomEdge.getVertex("vertex1");
Task t1= v1.getTask(0);
[48/50] [abbrv] tez git commit: TEZ-1721. Update INSTALL instructions
for clarifying tez client jars compatibility with runtime tarball on HDFS.
(hitesh)
Posted by je...@apache.org.
TEZ-1721. Update INSTALL instructions for clarifying tez client jars compatibility with runtime tarball on HDFS. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/dd03a811
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/dd03a811
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/dd03a811
Branch: refs/heads/TEZ-8
Commit: dd03a811598da8ac9175303845719413117b3d7a
Parents: 1a3059f
Author: Hitesh Shah <hi...@apache.org>
Authored: Fri Nov 14 15:47:33 2014 -0800
Committer: Hitesh Shah <hi...@apache.org>
Committed: Fri Nov 14 15:47:33 2014 -0800
----------------------------------------------------------------------
CHANGES.txt | 2 ++
docs/src/site/markdown/install.md | 4 ++++
2 files changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/dd03a811/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 0f66807..aa74705 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -10,6 +10,8 @@ ALL CHANGES:
TEZ-1738. Tez tfile parser for log parsing
TEZ-1627. Remove OUTPUT_CONSUMABLE and related Event in TaskAttemptImpl
TEZ-1736. Add support for Inputs/Outputs in runtime-library to generate history text data.
+ TEZ-1721. Update INSTALL instructions for clarifying tez client jars
+ compatibility with runtime tarball on HDFS.
Release 0.5.3: Unreleased
http://git-wip-us.apache.org/repos/asf/tez/blob/dd03a811/docs/src/site/markdown/install.md
----------------------------------------------------------------------
diff --git a/docs/src/site/markdown/install.md b/docs/src/site/markdown/install.md
index d31eeb9..ef49b5e 100644
--- a/docs/src/site/markdown/install.md
+++ b/docs/src/site/markdown/install.md
@@ -59,6 +59,10 @@ Replace x.y.z with the tez release number that you are using. E.g. 0.5.0
```
- Ensure tez.use.cluster.hadoop-libs is not set in tez-site.xml,
or if it is set, the value should be false
+ - Please note that the tarball version should match the version of
+ the client jars used when submitting Tez jobs to the cluster.
+ Please refer to the [Version Compatibility Guide](https://cwiki.apache.org/confluence/display/TEZ/Version+Compatibility)
+ for more details on version compatibility and detecting mismatches.
4. Optional: If running existing MapReduce jobs on Tez. Modify
mapred-site.xml to change "mapreduce.framework.name" property from
its default value of "yarn" to "yarn-tez"
[37/50] [abbrv] tez git commit: TEZ-1761.
TestRecoveryParser::testGetLastInProgressDAG fails in similar manner to
TEZ-1686. (zjffdu)
Posted by je...@apache.org.
TEZ-1761. TestRecoveryParser::testGetLastInProgressDAG fails in similar manner to TEZ-1686. (zjffdu)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/a7d55e43
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/a7d55e43
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/a7d55e43
Branch: refs/heads/TEZ-8
Commit: a7d55e43165cfc570fec6697d80e5ea73d04cf42
Parents: 67944a1
Author: Jeff Zhang <zj...@apache.org>
Authored: Sun Nov 9 11:54:08 2014 +0800
Committer: Jeff Zhang <zj...@apache.org>
Committed: Sun Nov 9 11:54:08 2014 +0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java | 4 ++--
2 files changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/a7d55e43/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 4f47fce..57a0c7f 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -20,6 +20,7 @@ ALL CHANGES:
TEZ-1746. Flaky test in TestVertexImpl and TestExceptionPropagation.
TEZ-1749. Increase test timeout for TestLocalMode.testMultipleClientsWithSession
TEZ-1750. Add a DAGScheduler which schedules tasks only when sources have been scheduled.
+ TEZ-1761. TestRecoveryParser::testGetLastInProgressDAG fails in similar manner to TEZ-1686.
Release 0.5.2: 2014-11-07
http://git-wip-us.apache.org/repos/asf/tez/blob/a7d55e43/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java
index 0425d25..6bbfa0f 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java
@@ -76,8 +76,8 @@ public class TestRecoveryParser {
public void testGetLastInProgressDAG() {
Map<TezDAGID, DAGSummaryData> summaryDataMap =
new HashMap<TezDAGID, DAGSummaryData>();
- int dagNum = 100;
- int lastInProgressDAGId = new Random().nextInt(dagNum);
+ int dagNum = 20;
+ int lastInProgressDAGId = new Random().nextInt(dagNum) + 1;
for (int i = 1; i <= dagNum; ++i) {
ApplicationId appId = ApplicationId.newInstance(1, 1);
TezDAGID dagId = TezDAGID.getInstance(appId, i);
[39/50] [abbrv] tez git commit: TEZ-1746. addendum. Increase the
timeout of TestExceptionPropagation.testExceptionPropagationSession
Posted by je...@apache.org.
TEZ-1746. addendum. Increase the timeout of TestExceptionPropagation.testExceptionPropagationSession
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/8f8bac84
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/8f8bac84
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/8f8bac84
Branch: refs/heads/TEZ-8
Commit: 8f8bac8430b7b2cba6bbada66724b61c890521b6
Parents: 6dda1d7
Author: Jeff Zhang <zj...@apache.org>
Authored: Tue Nov 11 09:12:51 2014 +0800
Committer: Jeff Zhang <zj...@apache.org>
Committed: Tue Nov 11 09:15:22 2014 +0800
----------------------------------------------------------------------
.../test/java/org/apache/tez/test/TestExceptionPropagation.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/8f8bac84/tez-tests/src/test/java/org/apache/tez/test/TestExceptionPropagation.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestExceptionPropagation.java b/tez-tests/src/test/java/org/apache/tez/test/TestExceptionPropagation.java
index efb6eb6..bb9888a 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/TestExceptionPropagation.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/TestExceptionPropagation.java
@@ -212,7 +212,7 @@ public class TestExceptionPropagation {
* @throws Exception
*
*/
- @Test(timeout = 300000)
+ @Test(timeout = 600000)
public void testExceptionPropagationSession() throws Exception {
try {
startSessionClient();
[18/50] [abbrv] tez git commit: TEZ-1735. Allow setting basic info
per DAG for Tez UI. (hitesh)
Posted by je...@apache.org.
TEZ-1735. Allow setting basic info per DAG for Tez UI. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/db4112cc
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/db4112cc
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/db4112cc
Branch: refs/heads/TEZ-8
Commit: db4112cc0019b36c587503ad5673b4a7265cc91a
Parents: 5782716
Author: Hitesh Shah <hi...@apache.org>
Authored: Mon Nov 3 19:55:57 2014 -0800
Committer: Hitesh Shah <hi...@apache.org>
Committed: Mon Nov 3 19:55:57 2014 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../main/java/org/apache/tez/dag/api/DAG.java | 26 +++++++++++++++++---
tez-api/src/main/proto/DAGApiRecords.proto | 1 +
.../apache/tez/dag/history/utils/DAGUtils.java | 4 +++
.../tez/dag/history/utils/TestDAGUtils.java | 4 +++
.../examples/TestOrderedWordCount.java | 1 +
6 files changed, 34 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/db4112cc/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index f741109..6fab873 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -98,6 +98,7 @@ ALL CHANGES:
TEZ-1726. Build broken against Hadoop-2.6.0 due to change in NodeReport.
TEZ-1579. MR examples should be setting mapreduce.framework.name to yarn-tez.
TEZ-1731. OnDiskMerger can end up clobbering files across tasks with LocalDiskFetch enabled.
+ TEZ-1735. Allow setting basic info per DAG for Tez UI.
Release 0.5.1: 2014-10-02
http://git-wip-us.apache.org/repos/asf/tez/blob/db4112cc/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
index 34c27bf..acef4da 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
@@ -48,7 +48,6 @@ import org.apache.tez.dag.api.EdgeProperty.DataMovementType;
import org.apache.tez.dag.api.EdgeProperty.DataSourceType;
import org.apache.tez.dag.api.EdgeProperty.SchedulingType;
import org.apache.tez.dag.api.VertexGroup.GroupInfo;
-import org.apache.tez.dag.api.TaskLocationHint;
import org.apache.tez.dag.api.records.DAGProtos.ConfigurationProto;
import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
import org.apache.tez.dag.api.records.DAGProtos.EdgePlan;
@@ -87,7 +86,8 @@ public class DAG {
Set<GroupInputEdge> groupInputEdges = Sets.newHashSet();
private DAGAccessControls dagAccessControls;
Map<String, LocalResource> commonTaskLocalFiles = Maps.newHashMap();
-
+ String dagInfo;
+
private Stack<String> topologicalVertexStack = new Stack<String>();
private DAG(String name) {
@@ -148,7 +148,24 @@ public class DAG {
this.credentials = credentials;
return this;
}
-
+
+ /**
+ * Set description info for this DAG that can be used for visualization purposes.
+ * @param dagInfo JSON blob as a serialized string.
+ * Recognized keys by the UI are:
+ * "context" - The application context in which this DAG is being used.
+ * For example, this could be set to "Hive" or "Pig" if
+ * this is being run as part of a Hive or Pig script.
+ * "description" - General description on what this DAG is going to do.
+ * In the case of Hive, this could be the SQL query text.
+ * @return {@link DAG}
+ */
+ public synchronized DAG setDAGInfo(String dagInfo) {
+ Preconditions.checkNotNull(dagInfo);
+ this.dagInfo = dagInfo;
+ return this;
+ }
+
/**
* Create a group of vertices that share a common output. This can be used to implement
* unions efficiently.
@@ -589,6 +606,9 @@ public class DAG {
DAGPlan.Builder dagBuilder = DAGPlan.newBuilder();
dagBuilder.setName(this.name);
+ if (this.dagInfo != null && !this.dagInfo.isEmpty()) {
+ dagBuilder.setDagInfo(this.dagInfo);
+ }
if (!vertexGroups.isEmpty()) {
for (VertexGroup av : vertexGroups) {
http://git-wip-us.apache.org/repos/asf/tez/blob/db4112cc/tez-api/src/main/proto/DAGApiRecords.proto
----------------------------------------------------------------------
diff --git a/tez-api/src/main/proto/DAGApiRecords.proto b/tez-api/src/main/proto/DAGApiRecords.proto
index 177faba..0539405 100644
--- a/tez-api/src/main/proto/DAGApiRecords.proto
+++ b/tez-api/src/main/proto/DAGApiRecords.proto
@@ -164,6 +164,7 @@ message DAGPlan {
optional bytes credentials_binary = 5;
repeated PlanVertexGroupInfo vertex_groups = 6;
repeated PlanLocalResource local_resource = 7;
+ optional string dag_info = 8;
}
// DAG monitoring messages
http://git-wip-us.apache.org/repos/asf/tez/blob/db4112cc/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
index 0bcbcbe..5f259989 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
@@ -49,6 +49,7 @@ import org.codehaus.jettison.json.JSONObject;
public class DAGUtils {
public static final String DAG_NAME_KEY = "dagName";
+ public static final String DAG_INFO_KEY = "dagInfo";
public static final String VERTICES_KEY = "vertices";
public static final String EDGES_KEY = "edges";
public static final String VERTEX_GROUPS_KEY = "vertexGroups";
@@ -138,6 +139,9 @@ public class DAGUtils {
final int version = 1;
Map<String,Object> dagMap = new LinkedHashMap<String, Object>();
dagMap.put(DAG_NAME_KEY, dagPlan.getName());
+ if (dagPlan.hasDagInfo()) {
+ dagMap.put(DAG_INFO_KEY, dagPlan.getDagInfo());
+ }
dagMap.put(VERSION_KEY, version);
ArrayList<Object> verticesList = new ArrayList<Object>();
for (DAGProtos.VertexPlan vertexPlan : dagPlan.getVertexList()) {
http://git-wip-us.apache.org/repos/asf/tez/blob/db4112cc/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java b/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java
index 6f4202a..496a12c 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java
@@ -71,6 +71,7 @@ public class TestDAGUtils {
dummyTaskCount, dummyTaskResource);
DAG dag = DAG.create("testDag");
+ dag.setDAGInfo("dagInfo");
String groupName1 = "uv12";
org.apache.tez.dag.api.VertexGroup uv12 = dag.createVertexGroup(groupName1, v1, v2);
OutputDescriptor outDesc = OutputDescriptor.create("output.class")
@@ -110,6 +111,9 @@ public class TestDAGUtils {
Map<String, Object> atsMap = DAGUtils.convertDAGPlanToATSMap(dagPlan);
Assert.assertTrue(atsMap.containsKey(DAGUtils.DAG_NAME_KEY));
+ Assert.assertEquals("testDag", atsMap.get(DAGUtils.DAG_NAME_KEY));
+ Assert.assertTrue(atsMap.containsKey(DAGUtils.DAG_INFO_KEY));
+ Assert.assertEquals("dagInfo", atsMap.get(DAGUtils.DAG_INFO_KEY));
Assert.assertEquals(dagPlan.getName(), atsMap.get(DAGUtils.DAG_NAME_KEY));
Assert.assertTrue(atsMap.containsKey("version"));
Assert.assertEquals(1, atsMap.get("version"));
http://git-wip-us.apache.org/repos/asf/tez/blob/db4112cc/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java
index e519085..ec6b429 100644
--- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java
@@ -270,6 +270,7 @@ public class TestOrderedWordCount extends Configured implements Tool {
vertices.add(finalReduceVertex);
DAG dag = DAG.create("OrderedWordCount" + dagIndex);
+ dag.setDAGInfo("{ \"context\": \"Tez\", \"description\": \"TestOrderedWordCount Job\" }");
for (int i = 0; i < vertices.size(); ++i) {
dag.addVertex(vertices.get(i));
}
[20/50] [abbrv] tez git commit: TEZ-1547. Make use of state change
notifier in VertexManagerPlugins and fix TEZ-1494 without latency penalty
(bikas)
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/tez/blob/cd0ed751/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
index bdcde73..97c0693 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
@@ -33,6 +33,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@@ -108,6 +109,7 @@ import org.apache.tez.dag.app.dag.Task;
import org.apache.tez.dag.app.dag.TaskAttemptStateInternal;
import org.apache.tez.dag.app.dag.Vertex;
import org.apache.tez.dag.app.dag.VertexState;
+import org.apache.tez.dag.app.dag.VertexStateUpdateListener;
import org.apache.tez.dag.app.dag.VertexTerminationCause;
import org.apache.tez.dag.app.dag.event.DAGEvent;
import org.apache.tez.dag.app.dag.event.DAGEventType;
@@ -2324,6 +2326,100 @@ public class TestVertexImpl {
.getOutputDescriptor().getClassName()));
}
+ class TestUpdateListener implements VertexStateUpdateListener {
+ List<VertexStateUpdate> events = Lists.newLinkedList();
+ @Override
+ public void onStateUpdated(VertexStateUpdate event) {
+ events.add(event);
+ }
+ }
+
+ @Test (timeout=5000)
+ public void testVertexConfigureEvent() throws Exception {
+ initAllVertices(VertexState.INITED);
+ TestUpdateListener listener = new TestUpdateListener();
+ updateTracker
+ .registerForVertexUpdates("vertex3",
+ EnumSet.of(org.apache.tez.dag.api.event.VertexState.CONFIGURED),
+ listener);
+ // completely configured event received for vertex when it inits
+ Assert.assertEquals(1, listener.events.size());
+ Assert.assertEquals("vertex3", listener.events.get(0).getVertexName());
+ Assert.assertEquals(org.apache.tez.dag.api.event.VertexState.CONFIGURED,
+ listener.events.get(0).getVertexState());
+ updateTracker.unregisterForVertexUpdates("vertex3", listener);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test (timeout=5000)
+ public void testVertexConfigureEventWithReconfigure() throws Exception {
+ useCustomInitializer = true;
+ setupPreDagCreation();
+ // initialize() will make VM call planned() and started() will make VM call done()
+ dagPlan = createDAGPlanWithVMException("TestVMStateUpdate", VMExceptionLocation.NoExceptionDoReconfigure);
+ setupPostDagCreation();
+
+ TestUpdateListener listener = new TestUpdateListener();
+ updateTracker
+ .registerForVertexUpdates("vertex2",
+ EnumSet.of(org.apache.tez.dag.api.event.VertexState.CONFIGURED),
+ listener);
+
+ VertexImplWithControlledInitializerManager v1 = (VertexImplWithControlledInitializerManager) vertices
+ .get("vertex1");
+ VertexImpl v2 = vertices.get("vertex2");
+ dispatcher.getEventHandler().handle(new VertexEvent(v1.getVertexId(),
+ VertexEventType.V_INIT));
+ // wait to ensure init is completed, so that rootInitManager is not null
+ dispatcher.await();
+ RootInputInitializerManagerControlled initializerManager1 = v1.getRootInputInitializerManager();
+ initializerManager1.completeInputInitialization();
+ dispatcher.await();
+ Assert.assertEquals(VertexState.INITED, v2.getState());
+ Assert.assertEquals(0, listener.events.size()); // configured event not sent
+ startVertex(v1, true);
+ Assert.assertEquals(VertexState.RUNNING, v2.getState());
+ Assert.assertEquals(1, listener.events.size()); // configured event sent after VM
+ Assert.assertEquals("vertex2", listener.events.get(0).getVertexName());
+ Assert.assertEquals(org.apache.tez.dag.api.event.VertexState.CONFIGURED,
+ listener.events.get(0).getVertexState());
+ updateTracker.unregisterForVertexUpdates("vertex2", listener);
+ }
+
+ @Test (timeout=5000)
+ public void testVertexConfigureEventBadReconfigure() {
+ initAllVertices(VertexState.INITED);
+ VertexImpl v3 = vertices.get("vertex3");
+ VertexImpl v2 = vertices.get("vertex2");
+ VertexImpl v1 = vertices.get("vertex1");
+ startVertex(v2);
+ startVertex(v1);
+ try {
+ // cannot call doneReconfiguringVertex() without calling vertexReconfigurationPlanned()
+ v3.doneReconfiguringVertex();
+ Assert.fail();
+ } catch (IllegalStateException e) {
+ Assert.assertTrue(e.getMessage().contains("invoked only after vertexReconfigurationPlanned"));
+ }
+ }
+
+ @Test (timeout=5000)
+ public void testVertexConfigureEventBadSetParallelism() throws Exception {
+ initAllVertices(VertexState.INITED);
+ VertexImpl v3 = vertices.get("vertex3");
+ VertexImpl v2 = vertices.get("vertex2");
+ VertexImpl v1 = vertices.get("vertex1");
+ startVertex(v2);
+ startVertex(v1);
+ try {
+ // cannot reconfigure a fully configured vertex without first notifying
+ v3.setParallelism(1, null, null, null, true);
+ Assert.fail();
+ } catch (IllegalStateException e) {
+ Assert.assertTrue(e.getMessage().contains("context.vertexReconfigurationPlanned() before re-configuring"));
+ }
+ }
+
@Test(timeout = 5000)
public void testVertexStart() {
initAllVertices(VertexState.INITED);
@@ -2333,6 +2429,19 @@ public class TestVertexImpl {
}
@Test(timeout = 5000)
+ public void testVertexReconfigurePlannedAfterInit() throws Exception {
+ initAllVertices(VertexState.INITED);
+ VertexImpl v3 = vertices.get("vertex3");
+
+ try {
+ v3.vertexReconfigurationPlanned();
+ Assert.fail();
+ } catch (IllegalStateException e) {
+ Assert.assertTrue(e.getMessage().contains("context.vertexReconfigurationPlanned() cannot be called after initialize()"));
+ }
+ }
+
+ @Test(timeout = 5000)
public void testVertexSetParallelism() throws Exception {
initAllVertices(VertexState.INITED);
VertexImpl v3 = vertices.get("vertex3");
@@ -2341,6 +2450,7 @@ public class TestVertexImpl {
Assert.assertEquals(2, tasks.size());
TezTaskID firstTask = tasks.keySet().iterator().next();
+ v3.vertexReconfigurationPlanned(true);
VertexImpl v1 = vertices.get("vertex1");
startVertex(vertices.get("vertex2"));
startVertex(v1);
@@ -2350,7 +2460,8 @@ public class TestVertexImpl {
Map<String, EdgeManagerPluginDescriptor> edgeManagerDescriptors =
Collections.singletonMap(
v1.getName(), mockEdgeManagerDescriptor);
- v3.setParallelism(1, null, edgeManagerDescriptors, null);
+ v3.setParallelism(1, null, edgeManagerDescriptors, null, true);
+ v3.doneReconfiguringVertex();
assertTrue(v3.sourceVertices.get(v1).getEdgeManager() instanceof
EdgeManagerForTest);
Assert.assertEquals(1, v3.getTotalTasks());
@@ -2368,15 +2479,19 @@ public class TestVertexImpl {
Map<TezTaskID, Task> tasks = v3.getTasks();
Assert.assertEquals(2, tasks.size());
+ v3.vertexReconfigurationPlanned(true);
VertexImpl v1 = vertices.get("vertex1");
startVertex(vertices.get("vertex2"));
startVertex(v1);
// increase not supported
try {
- v3.setParallelism(100, null, null, null);
+ v3.setParallelism(100, null, null, null, true);
+ v3.doneReconfiguringVertex();
Assert.fail();
- } catch (TezUncheckedException e) { }
+ } catch (TezUncheckedException e) {
+ Assert.assertTrue(e.getMessage().contains("Increasing parallelism is not supported"));
+ }
}
@Test(timeout = 5000)
@@ -2387,16 +2502,20 @@ public class TestVertexImpl {
Map<TezTaskID, Task> tasks = v3.getTasks();
Assert.assertEquals(2, tasks.size());
+ v3.vertexReconfigurationPlanned(true);
VertexImpl v1 = vertices.get("vertex1");
startVertex(vertices.get("vertex2"));
startVertex(v1);
- v3.setParallelism(1, null, null, null);
+ v3.setParallelism(1, null, null, null, true);
// multiple invocations not supported
try {
- v3.setParallelism(1, null, null, null);
+ v3.setParallelism(1, null, null, null, true);
Assert.fail();
- } catch (TezUncheckedException e) { }
+ } catch (TezUncheckedException e) {
+ Assert.assertTrue(e.getMessage().contains("Parallelism can only be set dynamically once per vertex"));
+ }
+ v3.doneReconfiguringVertex();
}
@SuppressWarnings("unchecked")
@@ -2451,12 +2570,14 @@ public class TestVertexImpl {
edgeManagerDescriptor.setUserPayload(userPayload);
Vertex v3 = vertices.get("vertex3");
- Vertex v5 = vertices.get("vertex5"); // Vertex5 linked to v3 (v3 src, v5
+ VertexImpl v5 = vertices.get("vertex5"); // Vertex5 linked to v3 (v3 src, v5
// dest)
+ v5.vertexReconfigurationPlanned(true);
Map<String, EdgeManagerPluginDescriptor> edgeManagerDescriptors =
Collections.singletonMap(v3.getName(), edgeManagerDescriptor);
- v5.setParallelism(v5.getTotalTasks() - 1, null, edgeManagerDescriptors, null); // Must decrease.
+ v5.setParallelism(v5.getTotalTasks() - 1, null, edgeManagerDescriptors, null, true); // Must decrease.
+ v5.doneReconfiguringVertex();
VertexImpl v5Impl = (VertexImpl) v5;
@@ -3096,8 +3217,8 @@ public class TestVertexImpl {
Assert.assertEquals(-1, v1.getTotalTasks());
Assert.assertEquals(VertexState.INITIALIZING, v1.getState());
// set the parallelism
- v1.setParallelism(numTasks, null, null, null);
- v2.setParallelism(numTasks, null, null, null);
+ v1.setParallelism(numTasks, null, null, null, true);
+ v2.setParallelism(numTasks, null, null, null, true);
dispatcher.await();
// parallelism set and vertex starts with pending start event
Assert.assertEquals(numTasks, v1.getTotalTasks());
@@ -3112,7 +3233,7 @@ public class TestVertexImpl {
// v3 still initializing with source vertex started. So should start running
// once num tasks is defined
Assert.assertEquals(VertexState.INITIALIZING, v3.getState());
- v3.setParallelism(numTasks, null, null, null);
+ v3.setParallelism(numTasks, null, null, null, false);
dispatcher.await();
Assert.assertEquals(numTasks, v3.getTotalTasks());
Assert.assertEquals(VertexState.RUNNING, v3.getState());
@@ -3223,8 +3344,9 @@ public class TestVertexImpl {
// fudge vertex manager so that tasks dont start running
v1.vertexManager = new VertexManager(
VertexManagerPluginDescriptor.create(VertexManagerPluginForTest.class.getName()),
- v1, appContext);
+ v1, appContext, mock(StateChangeNotifier.class));
v1.vertexManager.initialize();
+ v1.vertexReconfigurationPlanned(true);
startVertex(v1);
Assert.assertEquals(numTasks, vertices.get("vertex2").getTotalTasks());
@@ -3236,7 +3358,8 @@ public class TestVertexImpl {
Assert.assertEquals(VertexState.RUNNING, vertices.get("vertex4").getState());
// change parallelism
int newNumTasks = 3;
- v1.setParallelism(newNumTasks, null, null, null);
+ v1.setParallelism(newNumTasks, null, null, null, true);
+ v1.doneReconfiguringVertex();
dispatcher.await();
Assert.assertEquals(newNumTasks, vertices.get("vertex2").getTotalTasks());
Assert.assertEquals(newNumTasks, vertices.get("vertex3").getTotalTasks());
@@ -3260,7 +3383,7 @@ public class TestVertexImpl {
// fudge vertex manager so that tasks dont start running
v1.vertexManager = new VertexManager(
VertexManagerPluginDescriptor.create(VertexManagerPluginForTest.class.getName()),
- v1, appContext);
+ v1, appContext, mock(StateChangeNotifier.class));
v1.vertexManager.initialize();
Assert.assertEquals(numTasks, vertices.get("vertex2").getTotalTasks());
@@ -3268,7 +3391,9 @@ public class TestVertexImpl {
Assert.assertEquals(numTasks, vertices.get("vertex4").getTotalTasks());
// change parallelism
int newNumTasks = 3;
- v1.setParallelism(newNumTasks, null, null, null);
+ v1.vertexReconfigurationPlanned(true);
+ v1.setParallelism(newNumTasks, null, null, null, true);
+ v1.doneReconfiguringVertex();
dispatcher.await();
Assert.assertEquals(newNumTasks, vertices.get("vertex2").getTotalTasks());
Assert.assertEquals(newNumTasks, vertices.get("vertex3").getTotalTasks());
@@ -3414,12 +3539,14 @@ public class TestVertexImpl {
Assert.assertEquals(VertexState.SUCCEEDED, v1.getState());
- // At this point, 2 events should have been received - since the dispatcher is complete.
- Assert.assertEquals(2, initializer.stateUpdates.size());
- Assert.assertEquals(org.apache.tez.dag.api.event.VertexState.RUNNING,
+ // At this point, 3 events should have been received - since the dispatcher is complete.
+ Assert.assertEquals(3, initializer.stateUpdates.size());
+ Assert.assertEquals(org.apache.tez.dag.api.event.VertexState.CONFIGURED,
initializer.stateUpdates.get(0).getVertexState());
- Assert.assertEquals(org.apache.tez.dag.api.event.VertexState.SUCCEEDED,
+ Assert.assertEquals(org.apache.tez.dag.api.event.VertexState.RUNNING,
initializer.stateUpdates.get(1).getVertexState());
+ Assert.assertEquals(org.apache.tez.dag.api.event.VertexState.SUCCEEDED,
+ initializer.stateUpdates.get(2).getVertexState());
}
@Test(timeout = 10000)
@@ -3704,11 +3831,13 @@ public class TestVertexImpl {
Assert.assertEquals(0, initializerWrapper.getPendingEvents().get(v1.getName()).size());
Assert.assertTrue(initializer.eventReceived.get());
- Assert.assertEquals(2, initializer.stateUpdates.size());
- Assert.assertEquals(org.apache.tez.dag.api.event.VertexState.RUNNING,
+ Assert.assertEquals(3, initializer.stateUpdates.size());
+ Assert.assertEquals(org.apache.tez.dag.api.event.VertexState.CONFIGURED,
initializer.stateUpdates.get(0).getVertexState());
- Assert.assertEquals(org.apache.tez.dag.api.event.VertexState.SUCCEEDED,
+ Assert.assertEquals(org.apache.tez.dag.api.event.VertexState.RUNNING,
initializer.stateUpdates.get(1).getVertexState());
+ Assert.assertEquals(org.apache.tez.dag.api.event.VertexState.SUCCEEDED,
+ initializer.stateUpdates.get(2).getVertexState());
}
@Test(timeout = 10000)
@@ -3789,11 +3918,13 @@ public class TestVertexImpl {
// KK Add checks to validate thte RootInputManager doesn't remember the events either
Assert.assertTrue(initializer.eventReceived.get());
- Assert.assertEquals(2, initializer.stateUpdates.size());
- Assert.assertEquals(org.apache.tez.dag.api.event.VertexState.RUNNING,
+ Assert.assertEquals(3, initializer.stateUpdates.size());
+ Assert.assertEquals(org.apache.tez.dag.api.event.VertexState.CONFIGURED,
initializer.stateUpdates.get(0).getVertexState());
- Assert.assertEquals(org.apache.tez.dag.api.event.VertexState.SUCCEEDED,
+ Assert.assertEquals(org.apache.tez.dag.api.event.VertexState.RUNNING,
initializer.stateUpdates.get(1).getVertexState());
+ Assert.assertEquals(org.apache.tez.dag.api.event.VertexState.SUCCEEDED,
+ initializer.stateUpdates.get(2).getVertexState());
}
@SuppressWarnings("unchecked")
@@ -3873,6 +4004,7 @@ public class TestVertexImpl {
Assert.assertEquals(0, initializerWrapper.getPendingEvents().get(v1.getName()).size());
}
+ @SuppressWarnings("unchecked")
@Test(timeout = 5000)
/**
* Ref: TEZ-1494
@@ -3911,7 +4043,7 @@ public class TestVertexImpl {
assertTrue(m9.getState().equals(VertexState.INITED));
assertTrue(m5.getState().equals(VertexState.INITED));
assertTrue(m8.getState().equals(VertexState.INITED));
- assertTrue(m7.getVertexManager().getPlugin() instanceof ImmediateStartVertexManager);
+ assertTrue(m5.getVertexManager().getPlugin() instanceof ImmediateStartVertexManager);
//Start M9
dispatcher.getEventHandler().handle(new VertexEvent(m9.getVertexId(),
@@ -3921,41 +4053,13 @@ public class TestVertexImpl {
//Start M2; Let tasks complete in M2; Also let 1 task complete in R3
dispatcher.getEventHandler().handle(new VertexEvent(m2.getVertexId(), VertexEventType.V_START));
dispatcher.await();
- VertexEventTaskAttemptCompleted taskAttemptCompleted = new VertexEventTaskAttemptCompleted
- (TezTaskAttemptID.getInstance(TezTaskID.getInstance(m2.getVertexId(),0), 0), TaskAttemptStateInternal.SUCCEEDED);
- VertexEventTaskCompleted taskCompleted = new VertexEventTaskCompleted(TezTaskID.getInstance(m2
- .getVertexId(), 0), TaskState.SUCCEEDED);
- dispatcher.getEventHandler().handle(taskAttemptCompleted);
- dispatcher.getEventHandler().handle(taskCompleted);
- dispatcher.await();
- taskAttemptCompleted = new VertexEventTaskAttemptCompleted
- (TezTaskAttemptID.getInstance(TezTaskID.getInstance(r3.getVertexId(),0), 0),
- TaskAttemptStateInternal.SUCCEEDED);
- taskCompleted = new VertexEventTaskCompleted(TezTaskID.getInstance(r3
- .getVertexId(), 0), TaskState.SUCCEEDED);
- dispatcher.getEventHandler().handle(taskAttemptCompleted);
- dispatcher.getEventHandler().handle(taskCompleted);
- dispatcher.await();
- assertTrue(m2.getState().equals(VertexState.SUCCEEDED));
- assertTrue(m5.numSuccessSourceAttemptCompletions == 1);
- assertTrue(m5.getState().equals(VertexState.INITED));
- //R3 should be in running state as it has one task completed, and rest are pending
+ //R3 should be in running state
assertTrue(r3.getState().equals(VertexState.RUNNING));
//Let us start M7; M5 should start not start as it is dependent on M8 as well
dispatcher.getEventHandler().handle(new VertexEvent(m7.getVertexId(),VertexEventType.V_START));
dispatcher.await();
- //Let one of the tasks get over in M7 as well.
- taskAttemptCompleted = new VertexEventTaskAttemptCompleted
- (TezTaskAttemptID.getInstance(TezTaskID.getInstance(m7.getVertexId(),0), 0),
- TaskAttemptStateInternal.SUCCEEDED);
- taskCompleted = new VertexEventTaskCompleted(TezTaskID.getInstance(m7
- .getVertexId(), 0), TaskState.SUCCEEDED);
- dispatcher.getEventHandler().handle(taskAttemptCompleted);
- dispatcher.getEventHandler().handle(taskCompleted);
- dispatcher.await();
- assertTrue(m5.numSuccessSourceAttemptCompletions == 2);
//M5 should be in INITED state, as it depends on M8
assertTrue(m5.getState().equals(VertexState.INITED));
@@ -3969,25 +4073,9 @@ public class TestVertexImpl {
assertTrue(m9.getState().equals(VertexState.SUCCEEDED));
- //M5 in running state. But tasks should not be scheduled until M8 finishes a task.
+ //M5 in running state. All source vertices have started and are configured
assertTrue(m5.getState().equals(VertexState.RUNNING));
for(Task task : m5.getTasks().values()) {
- assertTrue(task.getState().equals(TaskState.NEW));
- }
-
- //Let one of the tasks get over in M8 as well. This should trigger tasks to be scheduled in M5
- taskAttemptCompleted = new VertexEventTaskAttemptCompleted
- (TezTaskAttemptID.getInstance(TezTaskID.getInstance(m8.getVertexId(),0), 0),
- TaskAttemptStateInternal.SUCCEEDED);
- taskCompleted = new VertexEventTaskCompleted(TezTaskID.getInstance(m8
- .getVertexId(), 0), TaskState.SUCCEEDED);
- dispatcher.getEventHandler().handle(taskAttemptCompleted);
- dispatcher.getEventHandler().handle(taskCompleted);
- dispatcher.await();
-
- assertTrue(m5.numSuccessSourceAttemptCompletions == 3);
- //Ensure all tasks in M5 are in scheduled state
- for(Task task : m5.getTasks().values()) {
assertTrue(task.getState().equals(TaskState.SCHEDULED));
}
}
@@ -4756,7 +4844,7 @@ public class TestVertexImpl {
Map<String, EdgeManagerPluginDescriptor> edges = Maps.newHashMap();
edges.put("B", mockEdgeManagerDescriptor);
- vC.setParallelism(2, vertexLocationHint, edges, null);
+ vC.setParallelism(2, vertexLocationHint, edges, null, true);
dispatcher.await();
Assert.assertEquals(VertexState.RUNNING, vA.getState());
@@ -4956,6 +5044,33 @@ public class TestVertexImpl {
Assert.assertTrue(diagnostics.contains(VMExceptionLocation.OnVertexManagerEventReceived.name()));
}
+ @SuppressWarnings("unchecked")
+ @Test(timeout = 5000)
+ public void testExceptionFromVM_OnVertexManagerVertexStateUpdated() throws AMUserCodeException {
+ useCustomInitializer = true;
+ setupPreDagCreation();
+ dagPlan = createDAGPlanWithVMException("TestVMStateUpdate", VMExceptionLocation.OnVertexManagerVertexStateUpdated);
+ setupPostDagCreation();
+
+ VertexImplWithControlledInitializerManager v1 = (VertexImplWithControlledInitializerManager) vertices
+ .get("vertex1");
+ VertexImpl v2 = vertices.get("vertex2");
+ dispatcher.getEventHandler().handle(new VertexEvent(v1.getVertexId(),
+ VertexEventType.V_INIT));
+ // wait to ensure init is completed, so that rootInitManager is not null
+ dispatcher.await();
+ RootInputInitializerManagerControlled initializerManager1 = v1.getRootInputInitializerManager();
+ initializerManager1.completeInputInitialization();
+ dispatcher.await();
+ Assert.assertEquals(VertexState.INITED, v2.getState());
+ startVertex(v1, false);
+
+ Assert.assertEquals(VertexState.FAILED, v2.getState());
+ String diagnostics = StringUtils.join(v2.getDiagnostics(), ",");
+ assertTrue(diagnostics.contains(VMExceptionLocation.OnVertexManagerVertexStateUpdated.name()));
+ Assert.assertEquals(VertexTerminationCause.AM_USERCODE_FAILURE, v2.getTerminationCause());
+ }
+
@Test(timeout = 5000)
public void testExceptionFromII_Initialize() throws AMUserCodeException {
useCustomInitializer = true;
@@ -5092,6 +5207,7 @@ public class TestVertexImpl {
initVertex(v1);
startVertex(v1); // v2 would get the state update from v1
+ dispatcher.await();
Assert.assertEquals(VertexState.RUNNING, v1.getState());
Assert.assertEquals(VertexState.FAILED, v2.getState());
String diagnostics = StringUtils.join(v2.getDiagnostics(), ",");
@@ -5215,10 +5331,12 @@ public class TestVertexImpl {
public static class VertexManagerWithException extends RootInputVertexManager{
public static enum VMExceptionLocation {
+ NoExceptionDoReconfigure,
OnRootVertexInitialized,
OnSourceTaskCompleted,
OnVertexStarted,
OnVertexManagerEventReceived,
+ OnVertexManagerVertexStateUpdated,
Initialize,
}
@@ -5236,6 +5354,9 @@ public class TestVertexImpl {
if (this.exLocation == VMExceptionLocation.Initialize) {
throw new RuntimeException(this.exLocation.name());
}
+ if (this.exLocation == VMExceptionLocation.NoExceptionDoReconfigure) {
+ getContext().vertexReconfigurationPlanned();
+ }
}
@Override
@@ -5261,6 +5382,9 @@ public class TestVertexImpl {
throw new RuntimeException(this.exLocation.name());
}
super.onVertexStarted(completions);
+ if (this.exLocation == VMExceptionLocation.NoExceptionDoReconfigure) {
+ getContext().doneReconfiguringVertex();
+ }
}
@Override
@@ -5270,6 +5394,15 @@ public class TestVertexImpl {
throw new RuntimeException(this.exLocation.name());
}
}
+
+ @Override
+ public void onVertexStateUpdated(VertexStateUpdate stateUpdate) {
+ super.onVertexStateUpdated(stateUpdate);
+ // depends on ImmediateStartVertexManager to register for state update
+ if (this.exLocation == VMExceptionLocation.OnVertexManagerVertexStateUpdated) {
+ throw new RuntimeException(this.exLocation.name());
+ }
+ }
}
public static enum IIExceptionLocation {
http://git-wip-us.apache.org/repos/asf/tez/blob/cd0ed751/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexManager.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexManager.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexManager.java
index ccd344b..5da6ce8 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexManager.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexManager.java
@@ -37,6 +37,7 @@ import org.apache.tez.dag.api.VertexManagerPlugin;
import org.apache.tez.dag.api.VertexManagerPluginContext;
import org.apache.tez.dag.api.VertexManagerPluginDescriptor;
import org.apache.tez.dag.app.AppContext;
+import org.apache.tez.dag.app.dag.StateChangeNotifier;
import org.apache.tez.dag.app.dag.Vertex;
import org.apache.tez.runtime.api.Event;
import org.apache.tez.runtime.api.events.InputDataInformationEvent;
@@ -61,7 +62,7 @@ public class TestVertexManager {
VertexManager vm =
new VertexManager(
VertexManagerPluginDescriptor.create(RootInputVertexManager.class
- .getName()), mockVertex, mockAppContext);
+ .getName()), mockVertex, mockAppContext, mock(StateChangeNotifier.class));
vm.initialize();
InputDescriptor id1 = mock(InputDescriptor.class);
List<Event> events1 = new LinkedList<Event>();
@@ -101,7 +102,7 @@ public class TestVertexManager {
VertexManager vm =
new VertexManager(
VertexManagerPluginDescriptor.create(CustomVertexManager.class
- .getName()), mockVertex, mockAppContext);
+ .getName()), mockVertex, mockAppContext, mock(StateChangeNotifier.class));
vm.initialize();
InputDescriptor id1 = mock(InputDescriptor.class);
List<Event> events1 = new LinkedList<Event>();
http://git-wip-us.apache.org/repos/asf/tez/blob/cd0ed751/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexRecovery.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexRecovery.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexRecovery.java
index b278d8f..8fa574c 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexRecovery.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexRecovery.java
@@ -27,6 +27,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -36,6 +37,7 @@ import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.util.SystemClock;
import org.apache.tez.common.counters.TezCounters;
+import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.api.records.DAGProtos;
import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
import org.apache.tez.dag.api.records.DAGProtos.EdgePlan;
@@ -51,6 +53,7 @@ import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.TaskAttemptListener;
import org.apache.tez.dag.app.TaskHeartbeatHandler;
import org.apache.tez.dag.app.dag.VertexState;
+import org.apache.tez.dag.app.dag.VertexTerminationCause;
import org.apache.tez.dag.app.dag.event.DAGEvent;
import org.apache.tez.dag.app.dag.event.DAGEventType;
import org.apache.tez.dag.app.dag.event.TaskAttemptEvent;
@@ -59,8 +62,10 @@ import org.apache.tez.dag.app.dag.event.TaskEvent;
import org.apache.tez.dag.app.dag.event.TaskEventRecoverTask;
import org.apache.tez.dag.app.dag.event.TaskEventType;
import org.apache.tez.dag.app.dag.event.VertexEvent;
+import org.apache.tez.dag.app.dag.event.VertexEventManagerUserCodeError;
import org.apache.tez.dag.app.dag.event.VertexEventRecoverVertex;
import org.apache.tez.dag.app.dag.event.VertexEventType;
+import org.apache.tez.dag.app.dag.impl.AMUserCodeException.Source;
import org.apache.tez.dag.app.dag.impl.TestVertexImpl.CountingOutputCommitter;
import org.apache.tez.dag.history.events.VertexRecoverableEventsGeneratedEvent;
import org.apache.tez.dag.history.events.VertexFinishedEvent;
@@ -72,6 +77,7 @@ import org.apache.tez.runtime.api.events.InputDataInformationEvent;
import org.apache.tez.runtime.api.impl.EventMetaData;
import org.apache.tez.runtime.api.impl.EventMetaData.EventProducerConsumerType;
import org.apache.tez.runtime.api.impl.TezEvent;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -649,6 +655,48 @@ public class TestVertexRecovery {
assertOutputCommitters(vertex3);
}
+
+
+ @Test
+ public void testRecovery_VertexManagerErrorOnRecovery() {
+ VertexImpl vertex1 = (VertexImpl) dag.getVertex("vertex1");
+ restoreFromInitializedEvent(vertex1);
+ vertex1.handle(new VertexEventRecoverVertex(vertex1.getVertexId(),
+ VertexState.RUNNING));
+ dispatcher.await();
+ assertEquals(VertexState.RUNNING, vertex1.getState());
+ assertEquals(vertex1.getTotalTasks(), vertex1.getTasks().size());
+ // verify OutputCommitter is initialized
+ assertOutputCommitters(vertex1);
+
+ VertexImpl vertex3 = (VertexImpl) dag.getVertex("vertex3");
+ VertexState recoveredState =
+ vertex3.restoreFromEvent(new VertexInitializedEvent(vertex3
+ .getVertexId(), "vertex3", initRequestedTime, initedTime, 0, "",
+ null));
+ assertEquals(VertexState.INITED, recoveredState);
+ // wait for recovery of vertex2
+ assertEquals(VertexState.RECOVERING, vertex3.getState());
+ vertex3.handle(new VertexEventManagerUserCodeError(vertex3.getVertexId(),
+ new AMUserCodeException(Source.VertexManager, new TezUncheckedException("test"))));
+
+ assertEquals(1, vertex3.numRecoveredSourceVertices);
+ assertEquals(1, vertex3.numInitedSourceVertices);
+ assertEquals(1, vertex3.numStartedSourceVertices);
+ assertEquals(1, vertex3.getDistanceFromRoot());
+ VertexImpl vertex2 = (VertexImpl) dag.getVertex("vertex2");
+ restoreFromInitializedEvent(vertex2);
+ vertex2.handle(new VertexEventRecoverVertex(vertex2.getVertexId(),
+ VertexState.RUNNING));
+ dispatcher.await();
+ assertEquals(VertexState.RUNNING, vertex2.getState());
+
+ // v3 FAILED due to user code error
+ assertEquals(VertexState.FAILED, vertex3.getState());
+ Assert.assertEquals(VertexTerminationCause.AM_USERCODE_FAILURE, vertex3.getTerminationCause());
+ assertEquals(2, vertex3.numRecoveredSourceVertices);
+ }
+
/**
* vertex1 (New) -> restoreFromInitialized -> StartRecoveryTransition<br>
http://git-wip-us.apache.org/repos/asf/tez/blob/cd0ed751/tez-examples/src/main/java/org/apache/tez/examples/SortMergeJoinExample.java
----------------------------------------------------------------------
diff --git a/tez-examples/src/main/java/org/apache/tez/examples/SortMergeJoinExample.java b/tez-examples/src/main/java/org/apache/tez/examples/SortMergeJoinExample.java
index 25d2a24..6b8dfea 100644
--- a/tez-examples/src/main/java/org/apache/tez/examples/SortMergeJoinExample.java
+++ b/tez-examples/src/main/java/org/apache/tez/examples/SortMergeJoinExample.java
@@ -43,6 +43,7 @@ import org.apache.tez.dag.api.TezException;
import org.apache.tez.dag.api.Vertex;
import org.apache.tez.dag.api.client.DAGClient;
import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.dag.library.vertexmanager.ShuffleVertexManager;
import org.apache.tez.examples.HashJoinExample.ForwardingProcessor;
import org.apache.tez.mapreduce.input.MRInput;
import org.apache.tez.mapreduce.output.MROutput;
@@ -250,13 +251,16 @@ public class SortMergeJoinExample extends Configured implements Tool {
* the join of the two sorted output from inputVertex1 and inputVerex2. It
* is load balanced across numPartitions.
*/
- Vertex joinVertex =
- Vertex.create(joiner,
- ProcessorDescriptor.create(SortMergeJoinProcessor.class.getName()),
- numPartitions).addDataSink(
+ Vertex joinVertex = Vertex
+ .create(joiner, ProcessorDescriptor.create(SortMergeJoinProcessor.class.getName()),
+ numPartitions)
+ .setVertexManagerPlugin(
+ ShuffleVertexManager.createConfigBuilder(tezConf).setAutoReduceParallelism(true)
+ .build())
+ .addDataSink(
joinOutput,
- MROutput.createConfigBuilder(new Configuration(tezConf),
- TextOutputFormat.class, outPath.toUri().toString()).build());
+ MROutput.createConfigBuilder(new Configuration(tezConf), TextOutputFormat.class,
+ outPath.toUri().toString()).build());
/**
* The output of inputVertex1 and inputVertex2 will be partitioned into
http://git-wip-us.apache.org/repos/asf/tez/blob/cd0ed751/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java
index 9f3d3f9..05f94c5 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java
@@ -26,6 +26,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.protobuf.ByteString;
import com.google.protobuf.InvalidProtocolBufferException;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Public;
@@ -44,6 +45,8 @@ import org.apache.tez.dag.api.VertexManagerPlugin;
import org.apache.tez.dag.api.VertexManagerPluginContext;
import org.apache.tez.dag.api.VertexManagerPluginContext.TaskWithLocationHint;
import org.apache.tez.dag.api.VertexManagerPluginDescriptor;
+import org.apache.tez.dag.api.event.VertexState;
+import org.apache.tez.dag.api.event.VertexStateUpdate;
import org.apache.tez.runtime.api.Event;
import org.apache.tez.runtime.api.events.DataMovementEvent;
import org.apache.tez.runtime.api.events.InputReadErrorEvent;
@@ -52,13 +55,16 @@ import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads.ShuffleEd
import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads.VertexManagerEventPayloadProto;
import javax.annotation.Nullable;
+
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.BitSet;
import java.util.Collections;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
/**
* Starts scheduling tasks when number of completed source tasks crosses
@@ -125,11 +131,12 @@ public class ShuffleVertexManager extends VertexManagerPlugin {
int totalNumBipartiteSourceTasks = 0;
int numBipartiteSourceTasksCompleted = 0;
int numVertexManagerEventsReceived = 0;
- List<Integer> pendingTasks;
+ List<Integer> pendingTasks = Lists.newLinkedList();
int totalTasksToSchedule = 0;
+ private AtomicBoolean onVertexStartedDone = new AtomicBoolean(false);
//Track source vertex and its finished tasks
- private final Map<String, SourceVertexInfo> srcVertexInfo = Maps.newHashMap();
+ private final Map<String, SourceVertexInfo> srcVertexInfo = Maps.newConcurrentMap();
boolean sourceVerticesScheduled = false;
@VisibleForTesting
int bipartiteSources = 0;
@@ -137,7 +144,7 @@ public class ShuffleVertexManager extends VertexManagerPlugin {
class SourceVertexInfo {
EdgeProperty edgeProperty;
- int numFinishedTasks;
+ boolean vertexIsConfigured;
BitSet finishedTaskSet;
SourceVertexInfo(EdgeProperty edgeProperty) {
@@ -321,8 +328,6 @@ public class ShuffleVertexManager extends VertexManagerPlugin {
@Override
public void onVertexStarted(Map<String, List<Integer>> completions) {
- pendingTasks = Lists.newArrayListWithCapacity(
- getContext().getVertexNumTasks(getContext().getVertexName()));
// track the tasks in this vertex
updatePendingTasks();
updateSourceTaskCount();
@@ -338,6 +343,7 @@ public class ShuffleVertexManager extends VertexManagerPlugin {
}
}
}
+ onVertexStartedDone.set(true);
// for the special case when source has 0 tasks or min fraction == 0
schedulePendingTasks();
}
@@ -346,7 +352,6 @@ public class ShuffleVertexManager extends VertexManagerPlugin {
public void onSourceTaskCompleted(String srcVertexName, Integer srcTaskId) {
updateSourceTaskCount();
SourceVertexInfo srcInfo = srcVertexInfo.get(srcVertexName);
- srcInfo.numFinishedTasks++;
if (srcInfo.edgeProperty.getDataMovementType() == DataMovementType.SCATTER_GATHER) {
//handle duplicate events for bipartite sources
@@ -524,6 +529,7 @@ public class ShuffleVertexManager extends VertexManagerPlugin {
//try to determine parallelism later when more info is available.
return;
}
+ getContext().doneReconfiguringVertex();
}
List<TaskWithLocationHint> scheduledTasks = Lists.newArrayListWithCapacity(numTasksToSchedule);
while(!pendingTasks.isEmpty() && numTasksToSchedule > 0) {
@@ -532,6 +538,10 @@ public class ShuffleVertexManager extends VertexManagerPlugin {
pendingTasks.remove(0);
}
getContext().scheduleVertexTasks(scheduledTasks);
+ if (pendingTasks.size() == 0) {
+ // done scheduling all tasks
+ // TODO TEZ-1714 locking issues. getContext().vertexManagerDone();
+ }
}
/**
@@ -542,13 +552,13 @@ public class ShuffleVertexManager extends VertexManagerPlugin {
boolean canScheduleTasks() {
for(Map.Entry<String, SourceVertexInfo> entry : srcVertexInfo.entrySet()) {
String sourceVertex = entry.getKey();
- int completedTasks = entry.getValue().numFinishedTasks;
int numSourceTasks = getContext().getVertexNumTasks(sourceVertex);
- if (LOG.isDebugEnabled()) {
- LOG.debug("vertex=" + getContext().getVertexName() + ", srcVertex="+ entry.getKey() +
- ", finishedTasks=" + entry.getValue().numFinishedTasks);
- }
- if (numSourceTasks > 0 && completedTasks <= 0) {
+ if (numSourceTasks > 0 && !entry.getValue().vertexIsConfigured) {
+ // vertex not configured
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Waiting for vertex: " + entry.getKey() + " in vertex: "
+ + getContext().getVertexName());
+ }
return false;
}
}
@@ -557,6 +567,10 @@ public class ShuffleVertexManager extends VertexManagerPlugin {
}
void schedulePendingTasks() {
+ if (!onVertexStartedDone.get()) {
+ // vertex not started yet
+ return;
+ }
int numPendingTasks = pendingTasks.size();
if (numPendingTasks == 0) {
return;
@@ -668,6 +682,8 @@ public class ShuffleVertexManager extends VertexManagerPlugin {
Map<String, EdgeProperty> inputs = getContext().getInputVertexEdgeProperties();
for(Map.Entry<String, EdgeProperty> entry : inputs.entrySet()) {
srcVertexInfo.put(entry.getKey(), new SourceVertexInfo(entry.getValue()));
+ getContext().registerForVertexStateUpdates(entry.getKey(),
+ EnumSet.of(VertexState.CONFIGURED));
if (entry.getValue().getDataMovementType() == DataMovementType.SCATTER_GATHER) {
bipartiteSources++;
}
@@ -675,12 +691,32 @@ public class ShuffleVertexManager extends VertexManagerPlugin {
if(bipartiteSources == 0) {
throw new TezUncheckedException("Atleast 1 bipartite source should exist");
}
+
+ if (enableAutoParallelism) {
+ getContext().vertexReconfigurationPlanned();
+ }
// dont track the source tasks here since those tasks may themselves be
// dynamically changed as the DAG progresses.
}
@Override
+ public void onVertexStateUpdated(VertexStateUpdate stateUpdate) {
+ Preconditions.checkArgument(stateUpdate.getVertexState() == VertexState.CONFIGURED,
+ "Received incorrect state notification : " + stateUpdate.getVertexState() + " for vertex: "
+ + stateUpdate.getVertexName() + " in vertex: " + getContext().getVertexName());
+ Preconditions.checkArgument(srcVertexInfo.containsKey(stateUpdate.getVertexName()),
+ "Received incorrect vertex notification : " + stateUpdate.getVertexState() + " for vertex: "
+ + stateUpdate.getVertexName() + " in vertex: " + getContext().getVertexName());
+ SourceVertexInfo vInfo = srcVertexInfo.get(stateUpdate.getVertexName());
+ Preconditions.checkState(vInfo.vertexIsConfigured == false);
+ vInfo.vertexIsConfigured = true;
+ LOG.info("Received configured notification : " + stateUpdate.getVertexState() + " for vertex: "
+ + stateUpdate.getVertexName() + " in vertex: " + getContext().getVertexName());
+ schedulePendingTasks();
+ }
+
+ @Override
public void onRootVertexInitialized(String inputName,
InputDescriptor inputDescriptor, List<Event> events) {
// Not allowing this for now. Nothing to do.
http://git-wip-us.apache.org/repos/asf/tez/blob/cd0ed751/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java
index 6e22194..76c0aa6 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java
@@ -36,6 +36,8 @@ import org.apache.tez.dag.api.VertexLocationHint;
import org.apache.tez.dag.api.VertexManagerPluginContext;
import org.apache.tez.dag.api.VertexManagerPluginContext.TaskWithLocationHint;
import org.apache.tez.dag.api.VertexManagerPluginDescriptor;
+import org.apache.tez.dag.api.event.VertexState;
+import org.apache.tez.dag.api.event.VertexStateUpdate;
import org.apache.tez.runtime.api.events.DataMovementEvent;
import org.apache.tez.runtime.api.events.VertexManagerEvent;
import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads.VertexManagerEventPayloadProto;
@@ -122,6 +124,7 @@ public class TestShuffleVertexManager {
manager = ReflectionUtils.createClazzInstance(pluginDesc.getClassName(),
new Class[]{VertexManagerPluginContext.class}, new Object[]{mockContext});
manager.initialize();
+ verify(mockContext, times(1)).vertexReconfigurationPlanned(); // Tez notified of reconfig
Assert.assertTrue(manager.enableAutoParallelism == true);
Assert.assertTrue(manager.desiredTaskInputDataSize == 1000l);
@@ -136,6 +139,7 @@ public class TestShuffleVertexManager {
manager = ReflectionUtils.createClazzInstance(pluginDesc.getClassName(),
new Class[]{VertexManagerPluginContext.class}, new Object[]{mockContext});
manager.initialize();
+ verify(mockContext, times(1)).vertexReconfigurationPlanned(); // Tez not notified of reconfig
Assert.assertTrue(manager.enableAutoParallelism == false);
Assert.assertTrue(manager.desiredTaskInputDataSize ==
@@ -147,10 +151,6 @@ public class TestShuffleVertexManager {
ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION_DEFAULT);
- // check initialization
- manager = createManager(conf, mockContext, 0.1f, 0.1f);
- Assert.assertTrue(manager.bipartiteSources == 2);
-
final HashSet<Integer> scheduledTasks = new HashSet<Integer>();
doAnswer(new Answer() {
public Object answer(InvocationOnMock invocation) {
@@ -210,18 +210,41 @@ public class TestShuffleVertexManager {
return null;
}}).when(mockContext).setVertexParallelism(eq(2), any(VertexLocationHint.class), anyMap(), anyMap());
+ // check initialization
+ manager = createManager(conf, mockContext, 0.1f, 0.1f); // Tez notified of reconfig
+ verify(mockContext, times(2)).vertexReconfigurationPlanned();
+ Assert.assertTrue(manager.bipartiteSources == 2);
+
+
// source vertices have 0 tasks.
when(mockContext.getVertexNumTasks(mockSrcVertexId1)).thenReturn(0);
when(mockContext.getVertexNumTasks(mockSrcVertexId2)).thenReturn(0);
when(mockContext.getVertexNumTasks(mockSrcVertexId3)).thenReturn(1);
+ // check waiting for notification before scheduling
manager.onVertexStarted(null);
Assert.assertFalse(manager.pendingTasks.isEmpty());
- Assert.assertTrue(scheduledTasks.size() == 0); // no tasks scheduled
- manager.onSourceTaskCompleted(mockSrcVertexId3, 0);
+ // source vertices have 0 tasks. so only 1 notification needed. triggers scheduling
+ manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId3, VertexState.CONFIGURED));
Assert.assertTrue(manager.pendingTasks.isEmpty());
+ verify(mockContext, times(1)).doneReconfiguringVertex(); // reconfig done
Assert.assertTrue(scheduledTasks.size() == 4); // all tasks scheduled
scheduledTasks.clear();
+ // TODO TEZ-1714 locking verify(mockContext, times(1)).vertexManagerDone(); // notified after scheduling all tasks
+
+ // check scheduling only after onVertexStarted
+ manager = createManager(conf, mockContext, 0.1f, 0.1f); // Tez notified of reconfig
+ verify(mockContext, times(3)).vertexReconfigurationPlanned();
+ Assert.assertTrue(manager.bipartiteSources == 2);
+ // source vertices have 0 tasks. so only 1 notification needed. does not trigger scheduling
+ manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId3, VertexState.CONFIGURED));
+ verify(mockContext, times(1)).doneReconfiguringVertex(); // reconfig done
+ Assert.assertTrue(scheduledTasks.size() == 0); // no tasks scheduled
+ manager.onVertexStarted(null);
+ verify(mockContext, times(2)).doneReconfiguringVertex(); // reconfig done
+ Assert.assertTrue(manager.pendingTasks.isEmpty());
+ Assert.assertTrue(scheduledTasks.size() == 4); // all tasks scheduled
+
when(mockContext.getVertexNumTasks(mockSrcVertexId1)).thenReturn(2);
when(mockContext.getVertexNumTasks(mockSrcVertexId2)).thenReturn(2);
@@ -231,23 +254,25 @@ public class TestShuffleVertexManager {
VertexManagerEvent vmEvent = VertexManagerEvent.create("Vertex", payload);
// parallelism not change due to large data size
manager = createManager(conf, mockContext, 0.1f, 0.1f);
+ verify(mockContext, times(4)).vertexReconfigurationPlanned(); // Tez notified of reconfig
manager.onVertexStarted(null);
Assert.assertTrue(manager.pendingTasks.size() == 4); // no tasks scheduled
Assert.assertTrue(manager.totalNumBipartiteSourceTasks == 4);
manager.onVertexManagerEventReceived(vmEvent);
- //1 task of every source vertex needs to be completed. Until then, we defer scheduling
+ manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId1, VertexState.CONFIGURED));
+ manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId2, VertexState.CONFIGURED));
manager.onSourceTaskCompleted(mockSrcVertexId1, new Integer(0));
verify(mockContext, times(0)).setVertexParallelism(anyInt(), any(VertexLocationHint.class), anyMap(), anyMap());
- Assert.assertEquals(4, manager.pendingTasks.size()); // no task scheduled
-
- //1 task of every source vertex needs to be completed. Until then, we defer scheduling
- manager.onSourceTaskCompleted(mockSrcVertexId2, new Integer(0));
+ verify(mockContext, times(2)).doneReconfiguringVertex();
+ // trigger scheduling
+ manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId3, VertexState.CONFIGURED));
verify(mockContext, times(0)).setVertexParallelism(anyInt(), any(VertexLocationHint.class), anyMap(), anyMap());
- manager.onSourceTaskCompleted(mockSrcVertexId3, new Integer(0));
+ verify(mockContext, times(3)).doneReconfiguringVertex(); // reconfig done
Assert.assertEquals(0, manager.pendingTasks.size()); // all tasks scheduled
Assert.assertEquals(4, scheduledTasks.size());
- Assert.assertEquals(2, manager.numBipartiteSourceTasksCompleted);
+ // TODO TEZ-1714 locking verify(mockContext, times(2)).vertexManagerDone(); // notified after scheduling all tasks
+ Assert.assertEquals(1, manager.numBipartiteSourceTasksCompleted);
Assert.assertEquals(5000L, manager.completedSourceTasksOutputSize);
/**
@@ -297,9 +322,10 @@ public class TestShuffleVertexManager {
verify(mockContext, times(1)).setVertexParallelism(eq(2), any(VertexLocationHint.class),
anyMap(),
anyMap());
+ manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId1, VertexState.CONFIGURED));
+ manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId2, VertexState.CONFIGURED));
+ manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId3, VertexState.CONFIGURED));
manager.onSourceTaskCompleted(mockSrcVertexId2, new Integer(0));
- //Need to have 1 task completed from all sources for this vertex
- manager.onSourceTaskCompleted(mockSrcVertexId3, new Integer(0));
Assert.assertEquals(1, manager.pendingTasks.size());
Assert.assertEquals(1, scheduledTasks.size());
Assert.assertEquals(2, manager.numBipartiteSourceTasksCompleted);
@@ -319,6 +345,9 @@ public class TestShuffleVertexManager {
//min/max fraction of 0.0/0.2
manager = createManager(conf, mockContext, 0.0f, 0.2f);
manager.onVertexStarted(null);
+ manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId1, VertexState.CONFIGURED));
+ manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId2, VertexState.CONFIGURED));
+ manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId3, VertexState.CONFIGURED));
Assert.assertEquals(40, manager.pendingTasks.size()); // no tasks scheduled
Assert.assertEquals(40, manager.totalNumBipartiteSourceTasks);
Assert.assertEquals(0, manager.numBipartiteSourceTasksCompleted);
@@ -334,7 +363,6 @@ public class TestShuffleVertexManager {
//send 8th event with payload size as 100
manager.onVertexManagerEventReceived(vmEvent);
manager.onSourceTaskCompleted(mockSrcVertexId2, new Integer(8));
- manager.onSourceTaskCompleted(mockSrcVertexId3, new Integer(0));
//Since max threshold (40 * 0.2 = 8) is met, vertex manager should determine parallelism
verify(mockContext, times(1)).setVertexParallelism(eq(4), any(VertexLocationHint.class),
anyMap(),
@@ -353,6 +381,9 @@ public class TestShuffleVertexManager {
manager = createManager(conf, mockContext, 0.5f, 0.5f);
manager.onVertexStarted(null);
+ manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId1, VertexState.CONFIGURED));
+ manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId2, VertexState.CONFIGURED));
+ manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId3, VertexState.CONFIGURED));
Assert.assertEquals(4, manager.pendingTasks.size()); // no tasks scheduled
Assert.assertEquals(4, manager.totalNumBipartiteSourceTasks);
// task completion from non-bipartite stage does nothing
@@ -528,16 +559,19 @@ public class TestShuffleVertexManager {
Assert.assertTrue(manager.totalNumBipartiteSourceTasks == 4);
Assert.assertTrue(manager.totalTasksToSchedule == 3);
Assert.assertTrue(manager.numBipartiteSourceTasksCompleted == 0);
- //Atleast 1 task should be complete in all sources
- manager.onSourceTaskCompleted(mockSrcVertexId1, new Integer(0));
- manager.onSourceTaskCompleted(mockSrcVertexId2, new Integer(0));
- manager.onSourceTaskCompleted(mockSrcVertexId3, new Integer(0));
+ // all source vertices need to be configured
+ manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId1, VertexState.CONFIGURED));
+ manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId2, VertexState.CONFIGURED));
+ manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId3, VertexState.CONFIGURED));
Assert.assertTrue(manager.pendingTasks.isEmpty());
Assert.assertTrue(scheduledTasks.size() == 3); // all tasks scheduled
// min, max > 0 and min == max
manager = createManager(conf, mockContext, 0.25f, 0.25f);
manager.onVertexStarted(null);
+ manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId1, VertexState.CONFIGURED));
+ manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId2, VertexState.CONFIGURED));
+ manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId3, VertexState.CONFIGURED));
Assert.assertTrue(manager.pendingTasks.size() == 3); // no tasks scheduled
Assert.assertTrue(manager.totalNumBipartiteSourceTasks == 4);
// task completion from non-bipartite stage does nothing
@@ -546,16 +580,16 @@ public class TestShuffleVertexManager {
Assert.assertTrue(manager.totalNumBipartiteSourceTasks == 4);
Assert.assertTrue(manager.numBipartiteSourceTasksCompleted == 0);
manager.onSourceTaskCompleted(mockSrcVertexId1, new Integer(0));
- //1 task has to be completed in every source vertex. Until then, defer scheduling
- Assert.assertFalse(manager.pendingTasks.isEmpty());
- manager.onSourceTaskCompleted(mockSrcVertexId2, new Integer(0));
Assert.assertTrue(manager.pendingTasks.isEmpty());
Assert.assertTrue(scheduledTasks.size() == 3); // all tasks scheduled
- Assert.assertTrue(manager.numBipartiteSourceTasksCompleted == 2);
+ Assert.assertTrue(manager.numBipartiteSourceTasksCompleted == 1);
// min, max > 0 and min == max == absolute max 1.0
manager = createManager(conf, mockContext, 1.0f, 1.0f);
manager.onVertexStarted(null);
+ manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId1, VertexState.CONFIGURED));
+ manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId2, VertexState.CONFIGURED));
+ manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId3, VertexState.CONFIGURED));
Assert.assertTrue(manager.pendingTasks.size() == 3); // no tasks scheduled
Assert.assertTrue(manager.totalNumBipartiteSourceTasks == 4);
// task completion from non-bipartite stage does nothing
@@ -580,6 +614,9 @@ public class TestShuffleVertexManager {
// min, max > 0 and min == max
manager = createManager(conf, mockContext, 1.0f, 1.0f);
manager.onVertexStarted(null);
+ manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId1, VertexState.CONFIGURED));
+ manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId2, VertexState.CONFIGURED));
+ manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId3, VertexState.CONFIGURED));
Assert.assertTrue(manager.pendingTasks.size() == 3); // no tasks scheduled
Assert.assertTrue(manager.totalNumBipartiteSourceTasks == 4);
// task completion from non-bipartite stage does nothing
@@ -604,6 +641,9 @@ public class TestShuffleVertexManager {
// min, max > and min < max
manager = createManager(conf, mockContext, 0.25f, 0.75f);
manager.onVertexStarted(null);
+ manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId1, VertexState.CONFIGURED));
+ manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId2, VertexState.CONFIGURED));
+ manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId3, VertexState.CONFIGURED));
Assert.assertTrue(manager.pendingTasks.size() == 3); // no tasks scheduled
Assert.assertTrue(manager.totalNumBipartiteSourceTasks == 4);
manager.onSourceTaskCompleted(mockSrcVertexId1, new Integer(0));
@@ -629,6 +669,9 @@ public class TestShuffleVertexManager {
// min, max > and min < max
manager = createManager(conf, mockContext, 0.25f, 1.0f);
manager.onVertexStarted(null);
+ manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId1, VertexState.CONFIGURED));
+ manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId2, VertexState.CONFIGURED));
+ manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId3, VertexState.CONFIGURED));
Assert.assertTrue(manager.pendingTasks.size() == 3); // no tasks scheduled
Assert.assertTrue(manager.totalNumBipartiteSourceTasks == 4);
manager.onSourceTaskCompleted(mockSrcVertexId1, new Integer(0));
@@ -649,7 +692,7 @@ public class TestShuffleVertexManager {
/**
- * Tasks should be scheduled only when at least 1 task from each source vertex is complete
+ * Tasks should be scheduled only when all source vertices are configured completely
*/
@Test(timeout = 5000)
public void test_Tez1649_with_scatter_gather_edges() {
@@ -763,6 +806,9 @@ public class TestShuffleVertexManager {
}}).when(mockContext_R2).scheduleVertexTasks(anyList());
manager.onVertexStarted(null);
+ manager.onVertexStateUpdated(new VertexStateUpdate(m2, VertexState.CONFIGURED));
+ manager.onVertexStateUpdated(new VertexStateUpdate(m3, VertexState.CONFIGURED));
+
manager.onVertexManagerEventReceived(vmEvent);
Assert.assertEquals(3, manager.pendingTasks.size()); // no tasks scheduled
Assert.assertEquals(9, manager.totalNumBipartiteSourceTasks);
@@ -789,8 +835,8 @@ public class TestShuffleVertexManager {
anyMap(),
anyMap());
- //1 Task completes in R1. Now things in R2 should be able to proceed.
- manager.onSourceTaskCompleted(r1, new Integer(0));
+ // complete configuration of r1 triggers the scheduling
+ manager.onVertexStateUpdated(new VertexStateUpdate(r1, VertexState.CONFIGURED));
verify(mockContext_R2, times(1)).setVertexParallelism(eq(1), any(VertexLocationHint.class),
anyMap(),
anyMap());
@@ -815,7 +861,8 @@ public class TestShuffleVertexManager {
Assert.assertTrue(manager.pendingTasks.size() == 3); // no tasks scheduled
Assert.assertTrue(manager.totalNumBipartiteSourceTasks == 3);
- //Send events for all tasks of m3.
+ // Only need completed configuration notification from m3
+ manager.onVertexStateUpdated(new VertexStateUpdate(m3, VertexState.CONFIGURED));
manager.onSourceTaskCompleted(m3, new Integer(0));
Assert.assertTrue(manager.pendingTasks.size() == 0); // all tasks scheduled
Assert.assertTrue(scheduledTasks.size() == 3);
@@ -885,6 +932,9 @@ public class TestShuffleVertexManager {
}}).when(mockContext).scheduleVertexTasks(anyList());
manager.onVertexStarted(null);
+ manager.onVertexStateUpdated(new VertexStateUpdate(r1, VertexState.CONFIGURED));
+ manager.onVertexStateUpdated(new VertexStateUpdate(m2, VertexState.CONFIGURED));
+
Assert.assertEquals(3, manager.pendingTasks.size()); // no tasks scheduled
Assert.assertEquals(3, manager.totalNumBipartiteSourceTasks);
Assert.assertEquals(0, manager.numBipartiteSourceTasksCompleted);
@@ -901,15 +951,17 @@ public class TestShuffleVertexManager {
Assert.assertTrue(manager.totalNumBipartiteSourceTasks == 3);
//Send an event for m2.
- manager.onSourceTaskCompleted(m3, new Integer(0));
+ manager.onVertexStateUpdated(new VertexStateUpdate(m3, VertexState.CONFIGURED));
Assert.assertTrue(manager.pendingTasks.size() == 0); // all tasks scheduled
Assert.assertTrue(scheduledTasks.size() == 3);
//Scenario when numBipartiteSourceTasksCompleted == totalNumBipartiteSourceTasks.
- //Still, wait for a task to be completed from other edges
+ //Still, wait for a configuration to be completed from other edges
scheduledTasks.clear();
manager = createManager(conf, mockContext, 0.001f, 0.001f);
manager.onVertexStarted(null);
+ manager.onVertexStateUpdated(new VertexStateUpdate(r1, VertexState.CONFIGURED));
+
when(mockContext.getInputVertexEdgeProperties()).thenReturn(mockInputVertices);
when(mockContext.getVertexName()).thenReturn(mockManagedVertexId);
when(mockContext.getVertexNumTasks(mockManagedVertexId)).thenReturn(3);
@@ -924,8 +976,8 @@ public class TestShuffleVertexManager {
manager.onSourceTaskCompleted(r1, new Integer(2));
//Tasks from non-scatter edges of m2 and m3 are not complete.
Assert.assertTrue(manager.pendingTasks.size() == 3); // no tasks scheduled
- manager.onSourceTaskCompleted(m2, new Integer(0));
- manager.onSourceTaskCompleted(m3, new Integer(0));
+ manager.onVertexStateUpdated(new VertexStateUpdate(m2, VertexState.CONFIGURED));
+ manager.onVertexStateUpdated(new VertexStateUpdate(m3, VertexState.CONFIGURED));
//Got an event from other edges. Schedule all
Assert.assertTrue(manager.pendingTasks.size() == 0); // all tasks scheduled
Assert.assertTrue(scheduledTasks.size() == 3);
@@ -944,6 +996,8 @@ public class TestShuffleVertexManager {
manager = createManager(conf, mockContext, 0.001f, 0.001f);
manager.onVertexStarted(null);
+ manager.onVertexStateUpdated(new VertexStateUpdate(r1, VertexState.CONFIGURED));
+
Assert.assertEquals(3, manager.pendingTasks.size()); // no tasks scheduled
Assert.assertEquals(3, manager.totalNumBipartiteSourceTasks);
Assert.assertEquals(0, manager.numBipartiteSourceTasksCompleted);
@@ -954,7 +1008,8 @@ public class TestShuffleVertexManager {
Assert.assertTrue(manager.pendingTasks.size() == 3); // no tasks scheduled
Assert.assertTrue(scheduledTasks.size() == 0);
- manager.onSourceTaskCompleted(m3, new Integer(1));
+ // event from m3 triggers scheduling. no need for m2 since it has 0 tasks
+ manager.onVertexStateUpdated(new VertexStateUpdate(m3, VertexState.CONFIGURED));
Assert.assertTrue(manager.pendingTasks.size() == 0); // all tasks scheduled
Assert.assertTrue(scheduledTasks.size() == 3);
@@ -970,6 +1025,7 @@ public class TestShuffleVertexManager {
when(mockContext.getVertexNumTasks(m3)).thenReturn(0); //broadcast
//Send 1 events for tasks of r1.
+ manager.onVertexStateUpdated(new VertexStateUpdate(r1, VertexState.CONFIGURED));
manager.onSourceTaskCompleted(r1, new Integer(0));
Assert.assertTrue(manager.pendingTasks.size() == 0); // all tasks scheduled
Assert.assertTrue(scheduledTasks.size() == 3);
http://git-wip-us.apache.org/repos/asf/tez/blob/cd0ed751/tez-tests/src/test/java/org/apache/tez/test/TestAMRecovery.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestAMRecovery.java b/tez-tests/src/test/java/org/apache/tez/test/TestAMRecovery.java
index 1d3799f..42d6f5c 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/TestAMRecovery.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/TestAMRecovery.java
@@ -543,8 +543,9 @@ public class TestAMRecovery {
@Override
public void onSourceTaskCompleted(String srcVertexName, Integer taskId) {
+ int dagAttemptNumber = getContext().getDAGAttemptNumber();
super.onSourceTaskCompleted(srcVertexName, taskId);
- if (getContext().getDAGAttemptNumber() == 1) {
+ if (dagAttemptNumber == 1) {
if (conf.getBoolean(FAIL_ON_PARTIAL_FINISHED, true)) {
if (taskId == 0) {
System.exit(-1);
@@ -647,8 +648,8 @@ public class TestAMRecovery {
@Override
public void onSourceTaskCompleted(String srcVertexName, Integer taskId) {
- super.onSourceTaskCompleted(srcVertexName, taskId);
int curAttempt = getContext().getDAGAttemptNumber();
+ super.onSourceTaskCompleted(srcVertexName, taskId);
int failOnAttempt = conf.getInt(FAIL_ON_ATTEMPT, 1);
LOG.info("failOnAttempt:" + failOnAttempt);
LOG.info("curAttempt:" + curAttempt);
[45/50] [abbrv] tez git commit: TEZ-1774. AppLaunched event for
Timeline does not have start time set. (hitesh)
Posted by je...@apache.org.
TEZ-1774. AppLaunched event for Timeline does not have start time set. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/f152a7f3
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/f152a7f3
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/f152a7f3
Branch: refs/heads/TEZ-8
Commit: f152a7f3bcca82a343b759c2921dbc8d93aaf175
Parents: 0daf2ba
Author: Hitesh Shah <hi...@apache.org>
Authored: Thu Nov 13 11:01:44 2014 -0800
Committer: Hitesh Shah <hi...@apache.org>
Committed: Thu Nov 13 11:01:44 2014 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../history/logging/ats/ATSHistoryLoggingService.java | 13 +++++++++----
.../logging/ats/HistoryEventTimelineConversion.java | 4 ++++
.../ats/TestHistoryEventTimelineConversion.java | 4 ++++
4 files changed, 18 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/f152a7f3/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index aa7d74d..e73ab9b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -10,6 +10,7 @@ ALL CHANGES:
TEZ-1738. Tez tfile parser for log parsing
TEZ-1627. Remove OUTPUT_CONSUMABLE and related Event in TaskAttemptImpl
TEZ-1736. Add support for Inputs/Outputs in runtime-library to generate history text data.
+ TEZ-1774. AppLaunched event for Timeline does not have start time set.
Release 0.5.3: Unreleased
http://git-wip-us.apache.org/repos/asf/tez/blob/f152a7f3/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java
index 99cb441..ce09a3f 100644
--- a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java
+++ b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java
@@ -257,10 +257,15 @@ public class ATSHistoryLoggingService extends HistoryLoggingService {
timelineClient.putEntities(entities);
if (response != null
&& !response.getErrors().isEmpty()) {
- TimelinePutError err = response.getErrors().get(0);
- if (err.getErrorCode() != 0) {
- LOG.warn("Could not post history events to ATS"
- + ", atsPutError=" + err.getErrorCode());
+ int count = response.getErrors().size();
+ for (int i = 0; i < count; ++i) {
+ TimelinePutError err = response.getErrors().get(i);
+ if (err.getErrorCode() != 0) {
+ LOG.warn("Could not post history event to ATS"
+ + ", atsPutError=" + err.getErrorCode()
+ + ", entityId=" + entities[i].getEntityId()
+ + ", eventType=" + events.get(i).getHistoryEvent().getEventType());
+ }
}
}
// Do nothing additional, ATS client library should handle throttling
http://git-wip-us.apache.org/repos/asf/tez/blob/f152a7f3/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
index 865915e..4b6d648 100644
--- a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
+++ b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
@@ -142,6 +142,8 @@ public class HistoryEventTimelineConversion {
atsEntity.addOtherInfo(ATSConstants.CONFIG,
DAGUtils.convertConfigurationToATSMap(event.getConf()));
+ atsEntity.setStartTime(event.getLaunchTime());
+
return atsEntity;
}
@@ -530,6 +532,8 @@ public class HistoryEventTimelineConversion {
initEvt.setTimestamp(event.getInitedTime());
atsEntity.addEvent(initEvt);
+ atsEntity.setStartTime(event.getInitedTime());
+
atsEntity.addOtherInfo(ATSConstants.VERTEX_NAME, event.getVertexName());
atsEntity.addOtherInfo(ATSConstants.INIT_REQUESTED_TIME, event.getInitRequestedTime());
atsEntity.addOtherInfo(ATSConstants.INIT_TIME, event.getInitedTime());
http://git-wip-us.apache.org/repos/asf/tez/blob/f152a7f3/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
index a40aa74..ce47820 100644
--- a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
+++ b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
@@ -205,6 +205,8 @@ public class TestHistoryEventTimelineConversion {
TimelineEntity timelineEntity = HistoryEventTimelineConversion.convertToTimelineEntity(event);
+ Assert.assertEquals(launchTime, timelineEntity.getStartTime().longValue());
+
Assert.assertEquals(EntityTypes.TEZ_APPLICATION.name(), timelineEntity.getEntityType());
Assert.assertEquals("tez_" + applicationId.toString(), timelineEntity.getEntityId());
@@ -405,6 +407,8 @@ public class TestHistoryEventTimelineConversion {
Assert.assertEquals(EntityTypes.TEZ_VERTEX_ID.name(), timelineEntity.getEntityType());
Assert.assertEquals(tezVertexID.toString(), timelineEntity.getEntityId());
+ Assert.assertEquals(initedTime, timelineEntity.getStartTime().longValue());
+
Assert.assertEquals(1, timelineEntity.getRelatedEntities().size());
Assert.assertTrue(
timelineEntity.getRelatedEntities().get(EntityTypes.TEZ_DAG_ID.name()).contains(
[36/50] [abbrv] tez git commit: TEZ-1750. Add a DAGScheduler which
schedules tasks only when sources have been scheduled. (sseth)
Posted by je...@apache.org.
TEZ-1750. Add a DAGScheduler which schedules tasks only when sources
have been scheduled. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/67944a1d
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/67944a1d
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/67944a1d
Branch: refs/heads/TEZ-8
Commit: 67944a1dff87faf3b8bf7ecf3afb1bcda6c43dda
Parents: a2d5768
Author: Siddharth Seth <ss...@apache.org>
Authored: Sat Nov 8 14:02:04 2014 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Sat Nov 8 14:02:04 2014 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/tez/dag/api/TezConfiguration.java | 8 +-
.../apache/tez/dag/app/dag/impl/DAGImpl.java | 9 +-
.../DAGSchedulerNaturalOrderControlled.java | 256 +++++++++++++
.../TestDAGSchedulerNaturalOrderControlled.java | 374 +++++++++++++++++++
5 files changed, 645 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/67944a1d/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 34ffc84..4f47fce 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -19,6 +19,7 @@ ALL CHANGES:
TEZ-1747. Increase test timeout for TestSecureShuffle.
TEZ-1746. Flaky test in TestVertexImpl and TestExceptionPropagation.
TEZ-1749. Increase test timeout for TestLocalMode.testMultipleClientsWithSession
+ TEZ-1750. Add a DAGScheduler which schedules tasks only when sources have been scheduled.
Release 0.5.2: 2014-11-07
http://git-wip-us.apache.org/repos/asf/tez/blob/67944a1d/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index d9003b3..6873863 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -339,6 +339,12 @@ public class TezConfiguration extends Configuration {
public static final String TEZ_AM_CLIENT_AM_PORT_RANGE =
TEZ_AM_PREFIX + "client.am.port-range";
+ /**
+ * String value. The class to be used for DAG Scheduling. Expert level setting.
+ */
+ public static final String TEZ_AM_DAG_SCHEDULER_CLASS = TEZ_AM_PREFIX + "dag.scheduler.class";
+ public static final String TEZ_AM_DAG_SCHEDULER_CLASS_DEFAULT =
+ "org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrder";
/** Int value. The amount of memory in MB to be used by the AppMaster */
public static final String TEZ_AM_RESOURCE_MEMORY_MB = TEZ_AM_PREFIX
@@ -367,7 +373,7 @@ public class TezConfiguration extends Configuration {
/**
* Int value. The maximum heartbeat interval between the AM and RM in milliseconds
* Increasing this reduces the communication between the AM and the RM and can
- * help in scaling up. Expert level setting. Expert level setting.
+ * help in scaling up. Expert level setting.
*/
public static final String TEZ_AM_RM_HEARTBEAT_INTERVAL_MS_MAX = TEZ_AM_PREFIX
+ "am-rm.heartbeat.interval-ms.max";
http://git-wip-us.apache.org/repos/asf/tez/blob/67944a1d/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index cddcbd5..f877eb4 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.tez.common.ATSConstants;
+import org.apache.tez.common.ReflectionUtils;
import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.DagTypeConverters;
import org.apache.tez.dag.api.EdgeProperty;
@@ -75,6 +76,7 @@ import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.TaskAttemptListener;
import org.apache.tez.dag.app.TaskHeartbeatHandler;
+import org.apache.tez.dag.app.dag.DAG;
import org.apache.tez.dag.app.dag.DAGReport;
import org.apache.tez.dag.app.dag.DAGScheduler;
import org.apache.tez.dag.app.dag.DAGState;
@@ -1333,8 +1335,11 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
}
private static void assignDAGScheduler(DAGImpl dag) {
- LOG.info("Using Natural order dag scheduler");
- dag.dagScheduler = new DAGSchedulerNaturalOrder(dag, dag.eventHandler);
+ String dagSchedulerClassName = dag.conf.get(TezConfiguration.TEZ_AM_DAG_SCHEDULER_CLASS,
+ TezConfiguration.TEZ_AM_DAG_SCHEDULER_CLASS_DEFAULT);
+ LOG.info("Using DAG Scheduler: " + dagSchedulerClassName);
+ dag.dagScheduler = ReflectionUtils.createClazzInstance(dagSchedulerClassName, new Class<?>[] {
+ DAG.class, EventHandler.class}, new Object[] {dag, dag.eventHandler});
}
private static VertexImpl createVertex(DAGImpl dag, String vertexName, int vId) {
http://git-wip-us.apache.org/repos/asf/tez/blob/67944a1d/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrderControlled.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrderControlled.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrderControlled.java
new file mode 100644
index 0000000..7cfbf5a
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrderControlled.java
@@ -0,0 +1,256 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.dag.impl;
+
+import java.util.BitSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import com.google.common.collect.LinkedListMultimap;
+import com.google.common.collect.ListMultimap;
+import com.google.common.collect.Lists;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.tez.dag.app.dag.DAG;
+import org.apache.tez.dag.app.dag.DAGScheduler;
+import org.apache.tez.dag.app.dag.TaskAttempt;
+import org.apache.tez.dag.app.dag.Vertex;
+import org.apache.tez.dag.app.dag.event.DAGEventSchedulerUpdate;
+import org.apache.tez.dag.app.dag.event.DAGEventSchedulerUpdateTAAssigned;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEventSchedule;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+
+/**
+ * Schedules task attempts belonging to downstream vertices only after all attempts belonging to
+ * upstream vertices have been scheduled. If there's a slow start or delayed start of a particular
+ * vertex, this ensures that downstream tasks are not started before this</p>
+ * Some future enhancements
+ * - consider cluster capacity - and be more aggressive about scheduling downstream tasks before
+ * upstream tasks have completed. </p>
+ * - generic slow start mechanism across all vertices - independent of the type of edges.
+ */
+@SuppressWarnings("rawtypes")
+public class DAGSchedulerNaturalOrderControlled implements DAGScheduler {
+
+ private static final Log LOG =
+ LogFactory.getLog(DAGSchedulerNaturalOrderControlled.class);
+
+ private final DAG dag;
+ private final EventHandler handler;
+
+ // Tracks pending events, in case they're not sent immediately.
+ private final ListMultimap<String, TaskAttemptEventSchedule> pendingEvents =
+ LinkedListMultimap.create();
+ // Tacks vertices for which no additional scheduling checks are required. Once in this list, the
+ // vertex is considered to be fully scheduled.
+ private final Set<String> scheduledVertices = new HashSet<String>();
+ // Tracks tasks scheduled for a vertex.
+ private final Map<String, BitSet> vertexScheduledTasks = new HashMap<String, BitSet>();
+
+ public DAGSchedulerNaturalOrderControlled(DAG dag, EventHandler dispatcher) {
+ this.dag = dag;
+ this.handler = dispatcher;
+ }
+
+ @Override
+ public void vertexCompleted(Vertex vertex) {
+ }
+
+ // TODO Does ordering matter - it currently depends on the order returned by vertex.getOutput*
+ @Override
+ public void scheduleTask(DAGEventSchedulerUpdate event) {
+ TaskAttempt attempt = event.getAttempt();
+ Vertex vertex = dag.getVertex(attempt.getVertexID());
+ int vertexDistanceFromRoot = vertex.getDistanceFromRoot();
+
+ // natural priority. Handles failures and retries.
+ int priorityLowLimit = (vertexDistanceFromRoot + 1) * 3;
+ int priorityHighLimit = priorityLowLimit - 2;
+
+ TaskAttemptEventSchedule attemptEvent = new TaskAttemptEventSchedule(
+ attempt.getID(), priorityLowLimit, priorityHighLimit);
+
+ taskAttemptSeen(vertex.getName(), attempt.getID());
+
+ if (vertexAlreadyScheduled(vertex)) {
+ // Vertex previously marked ready for scheduling.
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Scheduling " + attempt.getID() + " between priorityLow: " + priorityLowLimit
+ + " and priorityHigh: " + priorityHighLimit);
+ }
+ sendEvent(attemptEvent);
+ // A new taks coming in here could send us over the enough tasks scheduled limit.
+ processDownstreamVertices(vertex);
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Attempting to schedule vertex: " + vertex.getLogIdentifier() +
+ " due to schedule event");
+ }
+ boolean scheduled = trySchedulingVertex(vertex);
+ if (scheduled) {
+ LOG.info("Scheduled vertex: " + vertex.getLogIdentifier());
+ // If ready to be scheduled, send out pending events and the current event.
+ // Send events out for this vertex first. Then try scheduling downstream vertices.
+ sendEventsForVertex(vertex.getName());
+ sendEvent(attemptEvent);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Processing downstream vertices for vertex: " + vertex.getLogIdentifier());
+ }
+ processDownstreamVertices(vertex);
+ } else {
+ pendingEvents.put(vertex.getName(), attemptEvent);
+ }
+ }
+ }
+
+ private void taskAttemptSeen(String vertexName, TezTaskAttemptID taskAttemptID) {
+ BitSet scheduledTasks = vertexScheduledTasks.get(vertexName);
+ if (scheduledTasks == null) {
+ scheduledTasks = new BitSet();
+ vertexScheduledTasks.put(vertexName, scheduledTasks);
+ }
+ if (taskAttemptID != null) { // null for 0 task vertices
+ scheduledTasks.set(taskAttemptID.getTaskID().getId());
+ }
+ }
+
+ private void sendEventsForVertex(String vertexName) {
+ for (TaskAttemptEventSchedule event : pendingEvents.removeAll(vertexName)) {
+ sendEvent(event);
+ }
+ }
+
+ /* Checks whether this vertex has been marked as ready to go in the past */
+ private boolean vertexAlreadyScheduled(Vertex vertex) {
+ return scheduledVertices.contains(vertex.getName());
+ }
+
+ private boolean scheduledTasksForwarded(Vertex vertex) {
+ boolean canSchedule = false;
+ BitSet scheduledTasks = vertexScheduledTasks.get(vertex.getName());
+ if (scheduledTasks != null) {
+ if (scheduledTasks.cardinality() >= vertex.getTotalTasks()) {
+ canSchedule = true;
+ }
+ }
+ return canSchedule;
+ }
+
+ private void processDownstreamVertices(Vertex vertex) {
+ List<Vertex> newlyScheduledVertices = Lists.newLinkedList();
+ Map<Vertex, Edge> outputVertexEdgeMap = vertex.getOutputVertices();
+ for (Vertex destVertex : outputVertexEdgeMap.keySet()) {
+ if (vertexAlreadyScheduled(destVertex)) { // Nothing to do if already scheduled.
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Attempting to schedule vertex: " + destVertex.getLogIdentifier() +
+ " due to upstream event from " + vertex.getLogIdentifier());
+ }
+ boolean scheduled = trySchedulingVertex(destVertex);
+ if (scheduled) {
+ LOG.info("Scheduled vertex: " + destVertex.getLogIdentifier() +
+ " due to upstream event from " + vertex.getLogIdentifier());
+ sendEventsForVertex(destVertex.getName());
+ newlyScheduledVertices.add(destVertex);
+ }
+ }
+ }
+
+ // Try scheduling all downstream vertices which were scheduled in this run.
+ for (Vertex downStreamVertex : newlyScheduledVertices) {
+ processDownstreamVertices(downStreamVertex);
+ }
+ }
+
+ /* Process the specified vertex, and add it to the cache of scheduled vertices if it can be scheduled */
+ private boolean trySchedulingVertex(Vertex vertex) {
+ boolean canSchedule = true;
+ if (vertexScheduledTasks.get(vertex.getName()) == null) {
+ // No scheduled requests seen yet. Do not mark this as ready.
+ // 0 task vertices handled elsewhere.
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "No schedule requests for vertex: " + vertex.getLogIdentifier() + ", Not scheduling");
+ }
+ canSchedule = false;
+ } else {
+ Map<Vertex, Edge> inputVertexEdgeMap = vertex.getInputVertices();
+ if (inputVertexEdgeMap == null || inputVertexEdgeMap.isEmpty()) {
+ // Nothing to wait for. Go ahead and scheduled.
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("No sources for vertex: " + vertex.getLogIdentifier() + ", Scheduling now");
+ }
+ } else {
+ // Check if all sources are scheduled.
+ for (Vertex srcVertex : inputVertexEdgeMap.keySet()) {
+ if (scheduledTasksForwarded(srcVertex)) {
+ // Nothing to wait for. Go ahead and check the next source.
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Trying to schedule: " + vertex.getLogIdentifier() +
+ ", All tasks forwarded for srcVertex: " + srcVertex.getLogIdentifier() +
+ ", count: " + srcVertex.getTotalTasks());
+ }
+ } else {
+ // Special case for vertices with 0 tasks. 0 check is sufficient since parallelism cannot increase.
+ if (srcVertex.getTotalTasks() == 0) {
+ LOG.info(
+ "Vertex: " + srcVertex.getLogIdentifier() + " has 0 tasks. Marking as scheduled");
+ scheduledVertices.add(srcVertex.getName());
+ taskAttemptSeen(srcVertex.getName(), null);
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "Not all sources schedule requests complete while trying to schedule: " +
+ vertex.getLogIdentifier() + ", For source vertex: " +
+ srcVertex.getLogIdentifier() + ", Forwarded requests: " +
+ (vertexScheduledTasks.get(srcVertex.getName()) == null ? "null(0)" :
+ vertexScheduledTasks.get(srcVertex.getName()).cardinality()) +
+ " out of " + srcVertex.getTotalTasks());
+ }
+ canSchedule = false;
+ break;
+ }
+ }
+ }
+ }
+ }
+ if (canSchedule) {
+ scheduledVertices.add(vertex.getName());
+ }
+ return canSchedule;
+ }
+
+ @Override
+ public void taskScheduled(DAGEventSchedulerUpdateTAAssigned event) {
+ }
+
+ @Override
+ public void taskSucceeded(DAGEventSchedulerUpdate event) {
+ }
+
+ @SuppressWarnings("unchecked")
+ private void sendEvent(TaskAttemptEventSchedule event) {
+ handler.handle(event);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/67944a1d/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGSchedulerNaturalOrderControlled.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGSchedulerNaturalOrderControlled.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGSchedulerNaturalOrderControlled.java
new file mode 100644
index 0000000..88a91b6
--- /dev/null
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGSchedulerNaturalOrderControlled.java
@@ -0,0 +1,374 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.dag.impl;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.event.Event;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.tez.dag.app.dag.DAG;
+import org.apache.tez.dag.app.dag.TaskAttempt;
+import org.apache.tez.dag.app.dag.Vertex;
+import org.apache.tez.dag.app.dag.event.DAGEventSchedulerUpdate;
+import org.apache.tez.dag.app.dag.event.TaskAttemptEventSchedule;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+
+public class TestDAGSchedulerNaturalOrderControlled {
+
+ @Test(timeout = 5000)
+ public void testSimpleFlow() {
+ EventHandler eventHandler = mock(EventHandler.class);
+ DAG dag = createMockDag();
+ DAGSchedulerNaturalOrderControlled dagScheduler =
+ new DAGSchedulerNaturalOrderControlled(dag, eventHandler);
+
+ int numVertices = 5;
+ Vertex[] vertices = new Vertex[numVertices];
+ for (int i = 0; i < numVertices; i++) {
+ vertices[i] = dag.getVertex("vertex" + i);
+ }
+
+ // Schedule all tasks belonging to v0
+ for (int i = 0; i < vertices[0].getTotalTasks(); i++) {
+ dagScheduler.scheduleTask(createScheduleRequest(vertices[0].getVertexId(), i, 0));
+ }
+ verify(eventHandler, times(vertices[0].getTotalTasks())).handle(any(Event.class));
+ reset(eventHandler);
+
+ // Schedule 3 tasks belonging to v2
+ for (int i = 0; i < 3; i++) {
+ dagScheduler.scheduleTask(createScheduleRequest(vertices[2].getVertexId(), i, 0));
+ }
+ verify(eventHandler, times(3)).handle(any(Event.class));
+ reset(eventHandler);
+
+ // Schedule 3 tasks belonging to v3
+ for (int i = 0; i < 3; i++) {
+ dagScheduler.scheduleTask(createScheduleRequest(vertices[3].getVertexId(), i, 0));
+ }
+ verify(eventHandler, times(3)).handle(any(Event.class));
+ reset(eventHandler);
+
+ // Schedule remaining tasks belonging to v2
+ for (int i = 3; i < vertices[2].getTotalTasks(); i++) {
+ dagScheduler.scheduleTask(createScheduleRequest(vertices[2].getVertexId(), i, 0));
+ }
+ verify(eventHandler, times(vertices[2].getTotalTasks() - 3)).handle(any(Event.class));
+ reset(eventHandler);
+
+ // Schedule remaining tasks belonging to v3
+ for (int i = 3; i < vertices[3].getTotalTasks(); i++) {
+ dagScheduler.scheduleTask(createScheduleRequest(vertices[3].getVertexId(), i, 0));
+ }
+ verify(eventHandler, times(vertices[3].getTotalTasks() - 3)).handle(any(Event.class));
+ reset(eventHandler);
+
+
+ // Schedule all tasks belonging to v4
+ for (int i = 0; i < vertices[4].getTotalTasks(); i++) {
+ dagScheduler.scheduleTask(createScheduleRequest(vertices[4].getVertexId(), i, 0));
+ }
+ verify(eventHandler, times(vertices[4].getTotalTasks())).handle(any(Event.class));
+ reset(eventHandler);
+ }
+
+ @Test(timeout = 5000)
+ public void testSourceRequestDelayed() {
+ // ShuffleVertexHandler - slowstart simulation
+ EventHandler eventHandler = mock(EventHandler.class);
+ DAG dag = createMockDag();
+ DAGSchedulerNaturalOrderControlled dagScheduler =
+ new DAGSchedulerNaturalOrderControlled(dag, eventHandler);
+
+ int numVertices = 5;
+ Vertex[] vertices = new Vertex[numVertices];
+ for (int i = 0; i < numVertices; i++) {
+ vertices[i] = dag.getVertex("vertex" + i);
+ }
+
+ // Schedule all tasks belonging to v0
+ for (int i = 0; i < vertices[0].getTotalTasks(); i++) {
+ dagScheduler.scheduleTask(createScheduleRequest(vertices[0].getVertexId(), i, 0));
+ }
+ verify(eventHandler, times(vertices[0].getTotalTasks())).handle(any(Event.class));
+ reset(eventHandler);
+
+ // v2 behaving as if configured with slow-start.
+ // Schedule all tasks belonging to v3.
+ for (int i = 0; i < vertices[3].getTotalTasks(); i++) {
+ dagScheduler.scheduleTask(createScheduleRequest(vertices[3].getVertexId(), i, 0));
+ }
+ verify(eventHandler, times(vertices[3].getTotalTasks())).handle(any(Event.class));
+ reset(eventHandler);
+
+ // Scheduling all tasks belonging to v4. None should get scheduled.
+ for (int i = 0; i < vertices[4].getTotalTasks(); i++) {
+ dagScheduler.scheduleTask(createScheduleRequest(vertices[4].getVertexId(), i, 0));
+ }
+ verify(eventHandler, never()).handle(any(Event.class));
+ reset(eventHandler);
+
+ // v2 now starts scheduling ...
+ // Schedule 3 tasks for v2 initially.
+ for (int i = 0; i < 3; i++) {
+ dagScheduler.scheduleTask(createScheduleRequest(vertices[2].getVertexId(), i, 0));
+ }
+ verify(eventHandler, times(3)).handle(any(Event.class));
+ reset(eventHandler);
+
+ // Schedule remaining tasks belonging to v2
+ for (int i = 3; i < vertices[2].getTotalTasks(); i++) {
+ dagScheduler.scheduleTask(createScheduleRequest(vertices[2].getVertexId(), i, 0));
+ }
+ ArgumentCaptor<Event> args = ArgumentCaptor.forClass(Event.class);
+ // All of v2 and v3 should be sent out.
+ verify(eventHandler, times(vertices[2].getTotalTasks() - 3 + vertices[4].getTotalTasks()))
+ .handle(
+ args.capture());
+ int count = 0;
+ // Verify the order in which the events were sent out.
+ for (Event raw : args.getAllValues()) {
+ TaskAttemptEventSchedule event = (TaskAttemptEventSchedule) raw;
+ if (count < vertices[2].getTotalTasks() - 3) {
+ assertEquals(2, event.getTaskAttemptID().getTaskID().getVertexID().getId());
+ } else {
+ assertEquals(4, event.getTaskAttemptID().getTaskID().getVertexID().getId());
+ }
+ count++;
+ }
+ reset(eventHandler);
+ }
+
+
+ @Test(timeout = 5000)
+ public void testParallelismUpdated() {
+ EventHandler eventHandler = mock(EventHandler.class);
+ DAG dag = createMockDag();
+ DAGSchedulerNaturalOrderControlled dagScheduler =
+ new DAGSchedulerNaturalOrderControlled(dag, eventHandler);
+
+ int numVertices = 5;
+ Vertex[] vertices = new Vertex[numVertices];
+ for (int i = 0; i < numVertices; i++) {
+ vertices[i] = dag.getVertex("vertex" + i);
+ }
+
+ // Schedule all tasks belonging to v0
+ for (int i = 0; i < vertices[0].getTotalTasks(); i++) {
+ dagScheduler.scheduleTask(createScheduleRequest(vertices[0].getVertexId(), i, 0));
+ }
+ verify(eventHandler, times(vertices[0].getTotalTasks())).handle(any(Event.class));
+ reset(eventHandler);
+
+ assertEquals(10, vertices[2].getTotalTasks());
+
+ // v2 will change parallelism
+ // Schedule all tasks belonging to v3
+ for (int i = 0; i < vertices[3].getTotalTasks(); i++) {
+ dagScheduler.scheduleTask(createScheduleRequest(vertices[3].getVertexId(), i, 0));
+ }
+ verify(eventHandler, times(vertices[3].getTotalTasks())).handle(any(Event.class));
+ reset(eventHandler);
+
+ // Schedule all tasks belonging to v4
+ for (int i = 0; i < vertices[4].getTotalTasks(); i++) {
+ dagScheduler.scheduleTask(createScheduleRequest(vertices[4].getVertexId(), i, 0));
+ }
+ verify(eventHandler, never()).handle(any(Event.class));
+ reset(eventHandler);
+
+ // Reset the parallelism for v2.
+ updateParallelismOnMockVertex(vertices[2], 3);
+ assertEquals(3, vertices[2].getTotalTasks());
+
+ // Schedule all tasks belonging to v2
+ for (int i = 0; i < vertices[2].getTotalTasks(); i++) {
+ dagScheduler.scheduleTask(createScheduleRequest(vertices[2].getVertexId(), i, 0));
+ }
+ verify(eventHandler, times(vertices[2].getTotalTasks() + vertices[4].getTotalTasks()))
+ .handle(any(Event.class));
+ reset(eventHandler);
+ }
+
+ @Test(timeout = 5000)
+ public void testMultipleRequestsForSameTask() {
+ EventHandler eventHandler = mock(EventHandler.class);
+ DAG dag = createMockDag();
+ DAGSchedulerNaturalOrderControlled dagScheduler =
+ new DAGSchedulerNaturalOrderControlled(dag, eventHandler);
+
+ int numVertices = 5;
+ Vertex[] vertices = new Vertex[numVertices];
+ for (int i = 0; i < numVertices; i++) {
+ vertices[i] = dag.getVertex("vertex" + i);
+ }
+
+ // Schedule all but 1 task belonging to v0
+ for (int i = 0; i < vertices[0].getTotalTasks() - 1; i++) {
+ dagScheduler.scheduleTask(createScheduleRequest(vertices[0].getVertexId(), i, 0));
+ }
+ verify(eventHandler, times(vertices[0].getTotalTasks() - 1)).handle(any(Event.class));
+ reset(eventHandler);
+
+
+ // Schedule all tasks belonging to v2
+ for (int i = 0; i < vertices[2].getTotalTasks(); i++) {
+ dagScheduler.scheduleTask(createScheduleRequest(vertices[2].getVertexId(), i, 0));
+ }
+ // Nothing should be scheduled
+ verify(eventHandler, never()).handle(any(Event.class));
+ reset(eventHandler);
+
+ // Schedule an extra attempt for all but 1 task belonging to v0
+ for (int i = 0; i < vertices[0].getTotalTasks() - 1; i++) {
+ dagScheduler.scheduleTask(createScheduleRequest(vertices[0].getVertexId(), i, 1));
+ }
+ // Only v0 requests should have gone out
+ verify(eventHandler, times(vertices[0].getTotalTasks() - 1)).handle(any(Event.class));
+ reset(eventHandler);
+
+ // Schedule last task of v0, with attempt 1
+ dagScheduler.scheduleTask(
+ createScheduleRequest(vertices[0].getVertexId(), vertices[0].getTotalTasks() - 1, 1));
+ // One v0 request and all of v2 should have gone out
+ verify(eventHandler, times(1 + vertices[2].getTotalTasks())).handle(any(Event.class));
+ }
+
+
+ // Test parallelism updated form -1
+ // Reduce parallelism
+ // Different attempts scheduled for a single task.
+
+ private DAG createMockDag() {
+ DAG dag = mock(DAG.class);
+ /*
+ v0 v1
+ \ /
+ \ /
+ v2 v3
+ \ /
+ \ /
+ \ /
+ v4
+ v0 - Root
+ v1 - Root with 0 tasks.
+ v2 - can simulate AutoReduce. Parallelism goes down. Slow schedule.
+ v3 - can simulate ImmediateStart
+ v4 - Simulate one shuffle input, one broadcast input.
+ */
+
+ int numVertices = 5;
+ Vertex[] vertices = new Vertex[numVertices];
+
+ vertices[0] = createMockVertex("vertex0", 0, 10, 0);
+ vertices[1] = createMockVertex("vertex1", 1, 0, 0);
+ vertices[2] = createMockVertex("vertex2", 2, 10, 1);
+ vertices[3] = createMockVertex("vertex3", 3, 10, 1);
+ vertices[4] = createMockVertex("vertex4", 4, 10, 2);
+
+ for (int i = 0; i < numVertices; i++) {
+ String name = vertices[i].getName();
+ TezVertexID vertexId = vertices[i].getVertexId();
+ doReturn(vertices[i]).when(dag).getVertex(name);
+ doReturn(vertices[i]).when(dag).getVertex(vertexId);
+ }
+
+
+ updateMockVertexWithConnections(vertices[0], createConnectionMap(null),
+ createConnectionMap(vertices[2]));
+ updateMockVertexWithConnections(vertices[1], createConnectionMap(null),
+ createConnectionMap(vertices[3]));
+ updateMockVertexWithConnections(vertices[2], createConnectionMap(vertices[0]),
+ createConnectionMap(vertices[4]));
+ updateMockVertexWithConnections(vertices[3], createConnectionMap(vertices[1]),
+ createConnectionMap(vertices[4]));
+ updateMockVertexWithConnections(vertices[4], createConnectionMap(vertices[2], vertices[3]),
+ createConnectionMap(null));
+
+ return dag;
+ }
+
+
+ private void updateParallelismOnMockVertex(Vertex vertex, int newParallelism) {
+ doReturn(newParallelism).when(vertex).getTotalTasks();
+ }
+
+ private Vertex createMockVertex(String name, int vertexIdInt, int totalTasks,
+ int distanceFromRoot) {
+ ApplicationId appId = ApplicationId.newInstance(1000, 1);
+ TezDAGID dagId = TezDAGID.getInstance(appId, 1);
+ TezVertexID vertexId = TezVertexID.getInstance(dagId, vertexIdInt);
+
+ Vertex vertex = mock(Vertex.class);
+ doReturn(name).when(vertex).getName();
+ doReturn(totalTasks).when(vertex).getTotalTasks();
+ doReturn(vertexId).when(vertex).getVertexId();
+ doReturn(distanceFromRoot).when(vertex).getDistanceFromRoot();
+ doReturn(vertexId + " [" + name + "]").when(vertex).getLogIdentifier();
+ return vertex;
+ }
+
+ private Map<Vertex, Edge> createConnectionMap(Vertex... vertices) {
+ Map<Vertex, Edge> map = new HashMap<Vertex, Edge>();
+ if (vertices != null) {
+ for (Vertex vertex : vertices) {
+ map.put(vertex, mock(Edge.class));
+ }
+ }
+ return map;
+ }
+
+ private void updateMockVertexWithConnections(Vertex mockVertex, Map<Vertex, Edge> sources,
+ Map<Vertex, Edge> destinations) {
+ doReturn(sources).when(mockVertex).getInputVertices();
+ doReturn(destinations).when(mockVertex).getOutputVertices();
+ }
+
+ private TaskAttempt createTaskAttempt(TezVertexID vertexId, int taskIdInt, int attemptIdInt) {
+ TaskAttempt taskAttempt = mock(TaskAttempt.class);
+ TezTaskID taskId = TezTaskID.getInstance(vertexId, taskIdInt);
+ TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, attemptIdInt);
+ doReturn(taskAttemptId).when(taskAttempt).getID();
+ doReturn(vertexId).when(taskAttempt).getVertexID();
+ return taskAttempt;
+ }
+
+ private DAGEventSchedulerUpdate createScheduleRequest(TezVertexID vertexId, int taskIdInt,
+ int attemptIdInt) {
+ TaskAttempt mockAttempt = createTaskAttempt(vertexId, taskIdInt, attemptIdInt);
+ return new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt);
+ }
+
+}
[27/50] [abbrv] tez git commit: TEZ-1747. Increase test timeout for
TestSecureShuffle (Rajesh Balamohan)
Posted by je...@apache.org.
TEZ-1747. Increase test timeout for TestSecureShuffle (Rajesh Balamohan)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/7c6085d2
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/7c6085d2
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/7c6085d2
Branch: refs/heads/TEZ-8
Commit: 7c6085d24a2e8fa7a58b2fa32c47d283ca2b046f
Parents: 31c1de6
Author: Rajesh Balamohan <rb...@apache.org>
Authored: Thu Nov 6 18:21:24 2014 -0800
Committer: Rajesh Balamohan <rb...@apache.org>
Committed: Thu Nov 6 18:21:46 2014 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
tez-tests/src/test/java/org/apache/tez/test/TestSecureShuffle.java | 2 +-
2 files changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/7c6085d2/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 8fad485..e5a4eed 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -14,6 +14,7 @@ Release 0.5.3: Unreleased
ALL CHANGES:
TEZ-1745. TestATSHistoryLoggingService::testATSHistoryLoggingServiceShutdown can be flaky.
+ TEZ-1747. Increase test timeout for TestSecureShuffle.
Release 0.5.2: Unreleased
http://git-wip-us.apache.org/repos/asf/tez/blob/7c6085d2/tez-tests/src/test/java/org/apache/tez/test/TestSecureShuffle.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestSecureShuffle.java b/tez-tests/src/test/java/org/apache/tez/test/TestSecureShuffle.java
index 8cce736..37a8ef6 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/TestSecureShuffle.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/TestSecureShuffle.java
@@ -149,7 +149,7 @@ public class TestSecureShuffle {
*
* @throws Exception
*/
- @Test(timeout = 60000)
+ @Test(timeout = 240000)
public void testSecureShuffle() throws Exception {
//With tez-ssl setting
miniTezCluster.getConfig().setBoolean(
[12/50] [abbrv] tez git commit: TEZ-1703. addendum - fix flaky test.
(zjffdu)
Posted by je...@apache.org.
TEZ-1703. addendum - fix flaky test. (zjffdu)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/8f8a81f7
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/8f8a81f7
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/8f8a81f7
Branch: refs/heads/TEZ-8
Commit: 8f8a81f7a17f9018ae4e87bf0fca9d6cdc0a5ba4
Parents: 4d94652
Author: Jeff Zhang <zj...@apache.org>
Authored: Fri Oct 31 13:30:10 2014 +0800
Committer: Jeff Zhang <zj...@apache.org>
Committed: Fri Oct 31 13:30:10 2014 +0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../tez/dag/app/dag/impl/TestVertexImpl.java | 53 ++++++++------------
2 files changed, 21 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/8f8a81f7/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index cbbc0df..3fdd96e 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -94,6 +94,7 @@ ALL CHANGES:
TEZ-1711. Don't cache outputSpecList in VertexImpl.getOutputSpecList(taskIndex)
TEZ-1703. Exception handling for InputInitializer.
TEZ-1698. Cut down on ResourceCalculatorProcessTree overheads in Tez.
+ TEZ-1703. addendum - fix flaky test.
Release 0.5.1: 2014-10-02
http://git-wip-us.apache.org/repos/asf/tez/blob/8f8a81f7/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
index ef2c7bd..bdcde73 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
@@ -4974,28 +4974,27 @@ public class TestVertexImpl {
Assert.assertEquals(VertexState.FAILED, v1.getState());
Assert.assertEquals(VertexTerminationCause.ROOT_INPUT_INIT_FAILURE, v1.getTerminationCause());
}
-
+
@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testExceptionFromII_InitFailedAfterInitialized() throws AMUserCodeException {
useCustomInitializer = true;
- customInitializer = new EventHandlingRootInputInitializer(null, IIExceptionLocation.Initialize2);
- EventHandlingRootInputInitializer initializer =
- (EventHandlingRootInputInitializer) customInitializer;
setupPreDagCreation();
dagPlan = createDAGPlanWithIIException();
setupPostDagCreation();
- VertexImplWithRunningInputInitializer v1 =
- (VertexImplWithRunningInputInitializer) vertices.get("vertex1");
- // INIT_SUCCEEDED followed by INIT_FAILURE
+ VertexImplWithControlledInitializerManager v1 =
+ (VertexImplWithControlledInitializerManager)vertices.get("vertex1");
initVertex(v1);
- dispatcher.getEventHandler().handle(new VertexEventRootInputInitialized(
- v1.getVertexId(), "input1", null));
+ RootInputInitializerManagerControlled initializerManager1 = v1.getRootInputInitializerManager();
+ initializerManager1.completeInputInitialization(0);
+ Assert.assertEquals(VertexState.INITED, v1.getState());
+ String errorMsg = "ErrorWhenInitFailureAtInited";
+ dispatcher.getEventHandler().handle(new VertexEventRootInputFailed(v1.getVertexId(), "input1",
+ new AMUserCodeException(Source.InputInitializer, new Exception(errorMsg))));
dispatcher.await();
-
String diagnostics = StringUtils.join(v1.getDiagnostics(), ",");
- assertTrue(diagnostics.contains(IIExceptionLocation.Initialize2.name()));
+ assertTrue(diagnostics.contains(errorMsg));
Assert.assertEquals(VertexState.FAILED, v1.getState());
Assert.assertEquals(VertexTerminationCause.ROOT_INPUT_INIT_FAILURE, v1.getTerminationCause());
}
@@ -5004,24 +5003,23 @@ public class TestVertexImpl {
@Test(timeout = 5000)
public void testExceptionFromII_InitFailedAfterRunning() throws AMUserCodeException {
useCustomInitializer = true;
- customInitializer = new EventHandlingRootInputInitializer(null, IIExceptionLocation.Initialize2);
- EventHandlingRootInputInitializer initializer =
- (EventHandlingRootInputInitializer) customInitializer;
setupPreDagCreation();
dagPlan = createDAGPlanWithIIException();
setupPostDagCreation();
- VertexImplWithRunningInputInitializer v1 =
- (VertexImplWithRunningInputInitializer) vertices.get("vertex1");
+ VertexImplWithControlledInitializerManager v1 =
+ (VertexImplWithControlledInitializerManager)vertices.get("vertex1");
initVertex(v1);
- dispatcher.getEventHandler().handle(new VertexEventRootInputInitialized(
- v1.getVertexId(), "input1", null));
- dispatcher.getEventHandler().handle(new VertexEvent(v1.getVertexId(),
- VertexEventType.V_START));
+ RootInputInitializerManagerControlled initializerManager1 = v1.getRootInputInitializerManager();
+ initializerManager1.completeInputInitialization(0);
+ startVertex(v1);
+ Assert.assertEquals(VertexState.RUNNING, v1.getState());
+ String errorMsg = "ErrorWhenInitFailureAtRunning";
+ dispatcher.getEventHandler().handle(new VertexEventRootInputFailed(v1.getVertexId(), "input1",
+ new AMUserCodeException(Source.InputInitializer, new Exception(errorMsg))));
dispatcher.await();
-
String diagnostics = StringUtils.join(v1.getDiagnostics(), ",");
- assertTrue(diagnostics.contains(IIExceptionLocation.Initialize2.name()));
+ assertTrue(diagnostics.contains(errorMsg));
Assert.assertEquals(VertexState.FAILED, v1.getState());
Assert.assertEquals(VertexTerminationCause.ROOT_INPUT_INIT_FAILURE, v1.getTerminationCause());
}
@@ -5276,7 +5274,6 @@ public class TestVertexImpl {
public static enum IIExceptionLocation {
Initialize,
- Initialize2, // for test case that InputInitFailed after InputInitSucceeded
HandleInputInitializerEvent,
OnVertexStateUpdated
}
@@ -5314,16 +5311,6 @@ public class TestVertexImpl {
if (exLocation == IIExceptionLocation.Initialize) {
throw new Exception(exLocation.name());
}
- if (exLocation == IIExceptionLocation.Initialize2) {
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- // InputInitializerManager is been shutdown if Initialized succeeded,
- // catch the exception and throw the exception to simulate the case that
- // init failure after init succeeded
- throw new Exception(exLocation.name());
- }
- }
context.registerForVertexStateUpdates("vertex1", null);
initStarted.set(true);
lock.lock();
[11/50] [abbrv] tez git commit: TEZ-1698. Cut down on
ResourceCalculatorProcessTree overheads in Tez (Rajesh Balamohan & Gopal via
Rajesh Balamohan)
Posted by je...@apache.org.
TEZ-1698. Cut down on ResourceCalculatorProcessTree overheads in Tez (Rajesh Balamohan & Gopal via Rajesh Balamohan)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/4d94652c
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/4d94652c
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/4d94652c
Branch: refs/heads/TEZ-8
Commit: 4d94652c817d83204ae06d618ade07f15fbed8d7
Parents: 4ec2942
Author: Rajesh Balamohan <rb...@apache.org>
Authored: Fri Oct 31 08:12:21 2014 +0530
Committer: Rajesh Balamohan <rb...@apache.org>
Committed: Fri Oct 31 08:12:21 2014 +0530
----------------------------------------------------------------------
CHANGES.txt | 1 +
tez-plugins/pom.xml | 24 ++++++
.../tez-mbeans-resource-calculator/pom.xml | 60 +++++++++++++++
.../tez/util/TezMxBeanResourceCalculator.java | 77 ++++++++++++++++++++
.../util/TestTezMxBeanResourceCalculator.java | 74 +++++++++++++++++++
5 files changed, 236 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/4d94652c/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 61b380e..cbbc0df 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -93,6 +93,7 @@ ALL CHANGES:
TEZ-1722. DAG should be related to Application Id in ATS data.
TEZ-1711. Don't cache outputSpecList in VertexImpl.getOutputSpecList(taskIndex)
TEZ-1703. Exception handling for InputInitializer.
+ TEZ-1698. Cut down on ResourceCalculatorProcessTree overheads in Tez.
Release 0.5.1: 2014-10-02
http://git-wip-us.apache.org/repos/asf/tez/blob/4d94652c/tez-plugins/pom.xml
----------------------------------------------------------------------
diff --git a/tez-plugins/pom.xml b/tez-plugins/pom.xml
index b0ff489..7c15215 100644
--- a/tez-plugins/pom.xml
+++ b/tez-plugins/pom.xml
@@ -36,6 +36,30 @@
<module>tez-yarn-timeline-history</module>
</modules>
</profile>
+ <profile>
+ <id>jdk6check</id>
+ <activation>
+ <property>
+ <name>java.vendor.url</name>
+ <value>http://java.sun.com/</value>
+ </property>
+ </activation>
+ <modules>
+ <module>tez-mbeans-resource-calculator</module>
+ </modules>
+ </profile>
+ <profile>
+ <id>jdk7check</id>
+ <activation>
+ <property>
+ <name>java.vendor.url</name>
+ <value>http://java.oracle.com/</value>
+ </property>
+ </activation>
+ <modules>
+ <module>tez-mbeans-resource-calculator</module>
+ </modules>
+ </profile>
</profiles>
<build>
http://git-wip-us.apache.org/repos/asf/tez/blob/4d94652c/tez-plugins/tez-mbeans-resource-calculator/pom.xml
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-mbeans-resource-calculator/pom.xml b/tez-plugins/tez-mbeans-resource-calculator/pom.xml
new file mode 100644
index 0000000..bb1a0ce
--- /dev/null
+++ b/tez-plugins/tez-mbeans-resource-calculator/pom.xml
@@ -0,0 +1,60 @@
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-plugins</artifactId>
+ <version>0.6.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>tez-mbeans-resource-calculator</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-dag</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/tez/blob/4d94652c/tez-plugins/tez-mbeans-resource-calculator/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-mbeans-resource-calculator/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java b/tez-plugins/tez-mbeans-resource-calculator/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java
new file mode 100644
index 0000000..e8aeb0b
--- /dev/null
+++ b/tez-plugins/tez-mbeans-resource-calculator/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java
@@ -0,0 +1,77 @@
+package org.apache.tez.util;
+
+import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
+
+import java.lang.management.ManagementFactory;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Uses sun's MBeans to return process information.
+ */
+public class TezMxBeanResourceCalculator extends ResourceCalculatorProcessTree {
+
+ private final com.sun.management.OperatingSystemMXBean osBean;
+ private final Runtime runtime;
+ private final AtomicLong cumulativeCPU;
+
+ /**
+ * Create process-tree instance with specified root process.
+ * <p/>
+ * Subclass must override this.
+ *
+ * @param root process-tree root-process
+ */
+ public TezMxBeanResourceCalculator(String root) {
+ super(root);
+ runtime = Runtime.getRuntime();
+ osBean =
+ (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
+ cumulativeCPU = new AtomicLong();
+ }
+
+ @Override public void updateProcessTree() {
+ //nothing needs to be done as the data is read from OS mbeans.
+ }
+
+ @Override public String getProcessTreeDump() {
+ return "";
+ }
+
+ @Override public long getCumulativeVmem(int olderThanAge) {
+ return osBean.getCommittedVirtualMemorySize();
+ }
+
+ @Override public long getCumulativeRssmem(int olderThanAge) {
+ //Not supported directly (RSS ~= memory consumed by JVM from Xmx)
+ return runtime.totalMemory();
+ }
+
+ @Override public long getCumulativeCpuTime() {
+ //convert to milliseconds
+ return TimeUnit.MILLISECONDS.convert(cumulativeCPU.addAndGet(osBean.getProcessCpuTime()),
+ TimeUnit.MILLISECONDS);
+ }
+
+ @Override public boolean checkPidPgrpidForMatch() {
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/4d94652c/tez-plugins/tez-mbeans-resource-calculator/src/test/java/org/apache/tez/util/TestTezMxBeanResourceCalculator.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-mbeans-resource-calculator/src/test/java/org/apache/tez/util/TestTezMxBeanResourceCalculator.java b/tez-plugins/tez-mbeans-resource-calculator/src/test/java/org/apache/tez/util/TestTezMxBeanResourceCalculator.java
new file mode 100644
index 0000000..ade2118
--- /dev/null
+++ b/tez-plugins/tez-mbeans-resource-calculator/src/test/java/org/apache/tez/util/TestTezMxBeanResourceCalculator.java
@@ -0,0 +1,74 @@
+package org.apache.tez.util;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
+import org.apache.hadoop.yarn.client.api.TimelineClient;
+import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.app.AppContext;
+import org.apache.tez.dag.history.DAGHistoryEvent;
+import org.apache.tez.dag.history.events.DAGStartedEvent;
+import org.apache.tez.dag.records.TezDAGID;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class TestTezMxBeanResourceCalculator {
+
+ private ResourceCalculatorProcessTree resourceCalculator;
+
+ @Before
+ public void setup() throws Exception {
+ Configuration conf = new TezConfiguration();
+ conf.set(TezConfiguration.TEZ_TASK_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS,
+ TezMxBeanResourceCalculator.class.getName());
+
+ Class<? extends ResourceCalculatorProcessTree> clazz = conf.getClass(
+ TezConfiguration.TEZ_TASK_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS, null,
+ ResourceCalculatorProcessTree.class);
+ resourceCalculator = ResourceCalculatorProcessTree.getResourceCalculatorProcessTree(
+ System.getenv().get("JVM_PID"), clazz, conf);
+ }
+
+ @After
+ public void teardown() {
+ resourceCalculator = null;
+ }
+
+ @Test(timeout=5000)
+ public void testResourceCalculator() {
+ Assert.assertTrue(resourceCalculator instanceof TezMxBeanResourceCalculator);
+ Assert.assertTrue(resourceCalculator.getCumulativeCpuTime() > 0);
+ Assert.assertTrue(resourceCalculator.getCumulativeVmem() > 0);
+ Assert.assertTrue(resourceCalculator.getCumulativeRssmem() > 0);
+ Assert.assertTrue(resourceCalculator.getProcessTreeDump().equals(""));
+ Assert.assertTrue(resourceCalculator.checkPidPgrpidForMatch());
+ }
+
+}
[25/50] [abbrv] tez git commit: TEZ-1745.
TestATSHistoryLoggingService::testATSHistoryLoggingServiceShutdown can be
flaky. (hitesh)
Posted by je...@apache.org.
TEZ-1745. TestATSHistoryLoggingService::testATSHistoryLoggingServiceShutdown can be flaky. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/71ca74e0
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/71ca74e0
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/71ca74e0
Branch: refs/heads/TEZ-8
Commit: 71ca74e0e8e9e59dd1490961cc15482487cdbec8
Parents: 7b55e38
Author: Hitesh Shah <hi...@apache.org>
Authored: Thu Nov 6 16:45:41 2014 -0800
Committer: Hitesh Shah <hi...@apache.org>
Committed: Thu Nov 6 16:45:41 2014 -0800
----------------------------------------------------------------------
CHANGES.txt | 5 +++
.../logging/ats/ATSHistoryLoggingService.java | 8 ++++
.../ats/TestATSHistoryLoggingService.java | 47 +++++++++++++++-----
.../src/test/resources/log4j.properties | 19 ++++++++
4 files changed, 68 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/71ca74e0/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index adc1a85..2e58f0a 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,11 @@ ALL CHANGES:
TEZ-1733. TezMerger should sort FileChunks on size when merging
TEZ-1738. Tez tfile parser for log parsing
+Release 0.5.3: Unreleased
+
+ALL CHANGES:
+ TEZ-1745. TestATSHistoryLoggingService::testATSHistoryLoggingServiceShutdown can be flaky.
+
Release 0.5.2: Unreleased
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/tez/blob/71ca74e0/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java
index cdfbf7c..0108c26 100644
--- a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java
+++ b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java
@@ -119,6 +119,11 @@ public class ATSHistoryLoggingService extends HistoryLoggingService {
// Finish processing events and then return
interrupted = true;
}
+
+ if (events.isEmpty()) {
+ continue;
+ }
+
eventsProcessed += events.size();
try {
handleEvents(events);
@@ -242,6 +247,9 @@ public class ATSHistoryLoggingService extends HistoryLoggingService {
events.get(i).getHistoryEvent());
}
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Sending event batch to Timeline, batchSize=" + events.size());
+ }
try {
TimelinePutResponse response =
timelineClient.putEntities(entities);
http://git-wip-us.apache.org/repos/asf/tez/blob/71ca74e0/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryLoggingService.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryLoggingService.java b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryLoggingService.java
index 06bc065..a9e00c1 100644
--- a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryLoggingService.java
+++ b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryLoggingService.java
@@ -18,6 +18,8 @@
package org.apache.tez.dag.history.logging.ats;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
@@ -41,6 +43,8 @@ import static org.mockito.Mockito.when;
public class TestATSHistoryLoggingService {
+ private static final Log LOG = LogFactory.getLog(TestATSHistoryLoggingService.class);
+
private ATSHistoryLoggingService atsHistoryLoggingService;
private AppContext appContext;
private Configuration conf;
@@ -56,6 +60,7 @@ public class TestATSHistoryLoggingService {
conf = new Configuration(false);
conf.setLong(TezConfiguration.YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS,
1000l);
+ conf.setInt(TezConfiguration.YARN_ATS_MAX_EVENTS_PER_BATCH, 2);
atsInvokeCounter = 0;
atsEntitiesCounter = 0;
atsHistoryLoggingService.init(conf);
@@ -67,10 +72,10 @@ public class TestATSHistoryLoggingService {
new Answer<Object>() {
@Override
public Object answer(InvocationOnMock invocation) throws Throwable {
- atsEntitiesCounter += invocation.getArguments().length;
++atsInvokeCounter;
+ atsEntitiesCounter += invocation.getArguments().length;
try {
- Thread.sleep(500);
+ Thread.sleep(500l);
} catch (InterruptedException e) {
// do nothing
}
@@ -98,21 +103,41 @@ public class TestATSHistoryLoggingService {
}
try {
- Thread.sleep(500l);
+ Thread.sleep(2500l);
} catch (InterruptedException e) {
// Do nothing
}
atsHistoryLoggingService.stop();
- Assert.assertTrue("ATSEntitiesCounter = " + atsEntitiesCounter, atsEntitiesCounter >= 10);
- Assert.assertTrue("ATSEntitiesCounter = " + atsEntitiesCounter, atsEntitiesCounter < 50);
- Assert.assertTrue("ATSInvokeCounter = " + atsInvokeCounter, atsInvokeCounter >= 1);
- Assert.assertTrue("ATSInvokeCounter = " + atsInvokeCounter, atsInvokeCounter <= 10);
+ LOG.info("ATS entitiesSent=" + atsEntitiesCounter
+ + ", timelineInvocations=" + atsInvokeCounter);
+
+ Assert.assertTrue(atsEntitiesCounter >= 4);
+ Assert.assertTrue(atsEntitiesCounter < 20);
+
+ }
+
+ @Test(timeout=20000)
+ public void testATSEventBatching() {
+ TezDAGID tezDAGID = TezDAGID.getInstance(
+ ApplicationId.newInstance(100l, 1), 1);
+ DAGHistoryEvent historyEvent = new DAGHistoryEvent(tezDAGID,
+ new DAGStartedEvent(tezDAGID, 1001l, "user1", "dagName1"));
+
+ for (int i = 0; i < 100; ++i) {
+ atsHistoryLoggingService.handle(historyEvent);
+ }
+
+ try {
+ Thread.sleep(1000l);
+ } catch (InterruptedException e) {
+ // Do nothing
+ }
+ LOG.info("ATS entitiesSent=" + atsEntitiesCounter
+ + ", timelineInvocations=" + atsInvokeCounter);
- Assert.assertTrue("ATSInvokeCounter = " + atsInvokeCounter
- + ", ATSEntitiesCounter = " + atsEntitiesCounter,
- atsEntitiesCounter >=
- ((atsInvokeCounter-1) * TezConfiguration.YARN_ATS_MAX_EVENTS_PER_BATCH_DEFAULT));
+ Assert.assertTrue(atsEntitiesCounter > atsInvokeCounter);
+ Assert.assertEquals(atsEntitiesCounter/2, atsInvokeCounter);
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/71ca74e0/tez-plugins/tez-yarn-timeline-history/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history/src/test/resources/log4j.properties b/tez-plugins/tez-yarn-timeline-history/src/test/resources/log4j.properties
new file mode 100644
index 0000000..531b68b
--- /dev/null
+++ b/tez-plugins/tez-yarn-timeline-history/src/test/resources/log4j.properties
@@ -0,0 +1,19 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=info,stdout
+log4j.threshhold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n
[19/50] [abbrv] tez git commit: TEZ-1728. Remove local host name from
Fetcher thread name. (sseth)
Posted by je...@apache.org.
TEZ-1728. Remove local host name from Fetcher thread name. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/dfef97f6
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/dfef97f6
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/dfef97f6
Branch: refs/heads/TEZ-8
Commit: dfef97f6cf703d9c1c02f237ce17951f6bcc4a88
Parents: db4112c
Author: Siddharth Seth <ss...@apache.org>
Authored: Tue Nov 4 12:42:21 2014 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Tue Nov 4 12:42:21 2014 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../tez/runtime/library/common/shuffle/impl/ShuffleManager.java | 2 +-
.../common/shuffle/orderedgrouped/FetcherOrderedGrouped.java | 5 +----
3 files changed, 3 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/dfef97f6/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 6fab873..a321acf 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -99,6 +99,7 @@ ALL CHANGES:
TEZ-1579. MR examples should be setting mapreduce.framework.name to yarn-tez.
TEZ-1731. OnDiskMerger can end up clobbering files across tasks with LocalDiskFetch enabled.
TEZ-1735. Allow setting basic info per DAG for Tez UI.
+ TEZ-1728. Remove local host name from Fetcher thread name.
Release 0.5.1: 2014-10-02
http://git-wip-us.apache.org/repos/asf/tez/blob/dfef97f6/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java
index 75baa39..69c015e 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java
@@ -202,7 +202,7 @@ public class ShuffleManager implements FetcherCallback {
ExecutorService fetcherRawExecutor = Executors.newFixedThreadPool(
numFetchers,
new ThreadFactoryBuilder().setDaemon(true)
- .setNameFormat("Fetcher [" + srcNameTrimmed + "] #%d " + localhostName).build());
+ .setNameFormat("Fetcher [" + srcNameTrimmed + "] #%d").build());
this.fetcherExecutor = MoreExecutors.listeningDecorator(fetcherRawExecutor);
ExecutorService schedulerRawExecutor = Executors.newFixedThreadPool(1, new ThreadFactoryBuilder()
http://git-wip-us.apache.org/repos/asf/tez/blob/dfef97f6/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/FetcherOrderedGrouped.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/FetcherOrderedGrouped.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/FetcherOrderedGrouped.java
index e83c705..d51e45e 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/FetcherOrderedGrouped.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/FetcherOrderedGrouped.java
@@ -35,7 +35,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.compress.CompressionCodec;
-import org.apache.hadoop.net.NetUtils;
import org.apache.tez.common.TezUtilsInternal;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.tez.common.TezRuntimeFrameworkConfigs;
@@ -96,8 +95,6 @@ class FetcherOrderedGrouped extends Thread {
HttpConnection httpConnection;
HttpConnectionParams httpConnectionParams;
- final static String localhostName = NetUtils.getHostname();
-
// Initiative value is 0, which means it hasn't retried yet.
private long retryStartTime = 0;
@@ -142,7 +139,7 @@ class FetcherOrderedGrouped extends Thread {
this.localDiskFetchEnabled = localDiskFetchEnabled;
this.logIdentifier = "fetcher [" + TezUtilsInternal
- .cleanVertexName(inputContext.getSourceVertexName()) + "] #" + id + " " + localhostName;
+ .cleanVertexName(inputContext.getSourceVertexName()) + "] #" + id;
setName(logIdentifier);
setDaemon(true);
}
[07/50] [abbrv] tez git commit: TEZ-1716. Additional ATS data for UI.
(hitesh)
Posted by je...@apache.org.
TEZ-1716. Additional ATS data for UI. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/ede0e645
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/ede0e645
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/ede0e645
Branch: refs/heads/TEZ-8
Commit: ede0e645a9f4fdda44842a46b7fcf9edecdf50b2
Parents: a6c8006
Author: Hitesh Shah <hi...@apache.org>
Authored: Thu Oct 30 08:06:29 2014 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Thu Oct 30 08:06:29 2014 -0700
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/tez/common/ATSConstants.java | 5 +
.../org/apache/tez/dag/app/DAGAppMaster.java | 11 +-
.../apache/tez/dag/app/dag/impl/DAGImpl.java | 56 +++++-
.../apache/tez/dag/app/dag/impl/VertexImpl.java | 2 +
.../tez/dag/history/HistoryEventType.java | 1 +
.../dag/history/events/AppLaunchedEvent.java | 104 ++++++++++
.../dag/history/events/DAGFinishedEvent.java | 11 +-
.../dag/history/events/DAGInitializedEvent.java | 10 +-
.../dag/history/events/DAGSubmittedEvent.java | 8 +-
.../dag/history/events/TaskFinishedEvent.java | 1 +
.../tez/dag/history/logging/EntityTypes.java | 1 +
.../impl/HistoryEventJsonConversion.java | 51 ++++-
.../apache/tez/dag/history/utils/DAGUtils.java | 28 +--
.../tez/dag/app/dag/impl/TestDAGRecovery.java | 4 +-
.../TestHistoryEventsProtoConversion.java | 30 ++-
.../impl/TestHistoryEventJsonConversion.java | 12 +-
.../tez/dag/history/utils/TestDAGUtils.java | 5 +-
.../ats/HistoryEventTimelineConversion.java | 57 +++++-
.../ats/TestHistoryEventTimelineConversion.java | 188 ++++++++++++++++++-
20 files changed, 533 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/ede0e645/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index ca1da2e..5d6ebf5 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -89,6 +89,7 @@ ALL CHANGES:
invocations
TEZ-1700. Replace containerId from TaskLocationHint with [TaskIndex+Vertex]
based affinity
+ TEZ-1716. Additional ATS data for UI.
Release 0.5.1: 2014-10-02
http://git-wip-us.apache.org/repos/asf/tez/blob/ede0e645/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java b/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java
index ab81683..58761d5 100644
--- a/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java
+++ b/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java
@@ -50,9 +50,11 @@ public class ATSConstants {
public static final String APP_SUBMIT_TIME = "appSubmitTime";
/* Tez-specific info */
+ public static final String CONFIG = "config";
public static final String DAG_PLAN = "dagPlan";
public static final String DAG_NAME = "dagName";
public static final String VERTEX_NAME = "vertexName";
+ public static final String VERTEX_NAME_ID_MAPPING = "vertexNameIdMapping";
public static final String SCHEDULED_TIME = "scheduledTime";
public static final String INIT_REQUESTED_TIME = "initRequestedTime";
public static final String INIT_TIME = "initTime";
@@ -62,6 +64,7 @@ public class ATSConstants {
public static final String TIME_TAKEN = "timeTaken";
public static final String STATUS = "status";
public static final String DIAGNOSTICS = "diagnostics";
+ public static final String SUCCESSFUL_ATTEMPT_ID = "successfulAttemptId";
public static final String COUNTERS = "counters";
public static final String STATS = "stats";
public static final String NUM_TASKS = "numTasks";
@@ -70,6 +73,8 @@ public class ATSConstants {
public static final String NUM_SUCCEEDED_TASKS = "numSucceededTasks";
public static final String NUM_FAILED_TASKS = "numFailedTasks";
public static final String NUM_KILLED_TASKS = "numKilledTasks";
+ public static final String NUM_FAILED_TASKS_ATTEMPTS = "numFailedTaskAttempts";
+ public static final String NUM_KILLED_TASKS_ATTEMPTS = "numKilledTaskAttempts";
public static final String PROCESSOR_CLASS_NAME = "processorClassName";
public static final String IN_PROGRESS_LOGS_URL = "inProgressLogsURL";
public static final String COMPLETED_LOGS_URL = "completedLogsURL";
http://git-wip-us.apache.org/repos/asf/tez/blob/ede0e645/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 20da85b..789de24 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -145,6 +145,7 @@ import org.apache.tez.dag.history.DAGHistoryEvent;
import org.apache.tez.dag.history.HistoryEventHandler;
import org.apache.tez.dag.history.events.AMLaunchedEvent;
import org.apache.tez.dag.history.events.AMStartedEvent;
+import org.apache.tez.dag.history.events.AppLaunchedEvent;
import org.apache.tez.dag.history.events.DAGSubmittedEvent;
import org.apache.tez.dag.history.utils.DAGUtils;
import org.apache.tez.dag.records.TezDAGID;
@@ -459,6 +460,12 @@ public class DAGAppMaster extends AbstractService {
super.serviceInit(conf);
if (!versionMismatch) {
+ if (this.appAttemptID.getAttemptId() == 1) {
+ AppLaunchedEvent appLaunchedEvent = new AppLaunchedEvent(appAttemptID.getApplicationId(),
+ startTime, appSubmitTime, appMasterUgi.getShortUserName(), this.amConf);
+ historyEventHandler.handle(
+ new DAGHistoryEvent(appLaunchedEvent));
+ }
AMLaunchedEvent launchedEvent = new AMLaunchedEvent(appAttemptID,
startTime, appSubmitTime, appMasterUgi.getShortUserName());
historyEventHandler.handle(
@@ -749,7 +756,7 @@ public class DAGAppMaster extends AbstractService {
if (LOG.isDebugEnabled()) {
LOG.info("JSON dump for submitted DAG, dagId=" + dagId.toString()
+ ", json="
- + DAGUtils.generateSimpleJSONPlan(dagPB, newDag.getVertexNameIDMapping()).toString());
+ + DAGUtils.generateSimpleJSONPlan(dagPB).toString());
}
} catch (JSONException e) {
LOG.warn("Failed to generate json for DAG", e);
@@ -1915,7 +1922,7 @@ public class DAGAppMaster extends AbstractService {
// for an app later
DAGSubmittedEvent submittedEvent = new DAGSubmittedEvent(newDAG.getID(),
submitTime, dagPlan, this.appAttemptID, cumulativeAdditionalResources,
- newDAG.getUserName(), newDAG.getVertexNameIDMapping());
+ newDAG.getUserName());
try {
historyEventHandler.handleCriticalEvent(
new DAGHistoryEvent(newDAG.getID(), submittedEvent));
http://git-wip-us.apache.org/repos/asf/tez/blob/ede0e645/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index d3aecd4..6dccf3a 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
import org.apache.hadoop.yarn.util.Clock;
+import org.apache.tez.common.ATSConstants;
import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.DagTypeConverters;
import org.apache.tez.dag.api.EdgeProperty;
@@ -716,6 +717,40 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
}
}
+ private ProgressBuilder getDAGProgress() {
+ int totalTaskCount = 0;
+ int totalSucceededTaskCount = 0;
+ int totalRunningTaskCount = 0;
+ int totalFailedTaskCount = 0;
+ int totalKilledTaskCount = 0;
+ int totalFailedTaskAttemptCount = 0;
+ int totalKilledTaskAttemptCount = 0;
+ readLock.lock();
+ try {
+ for(Map.Entry<String, Vertex> entry : vertexMap.entrySet()) {
+ ProgressBuilder progress = entry.getValue().getVertexProgress();
+ totalTaskCount += progress.getTotalTaskCount();
+ totalSucceededTaskCount += progress.getSucceededTaskCount();
+ totalRunningTaskCount += progress.getRunningTaskCount();
+ totalFailedTaskCount += progress.getFailedTaskCount();
+ totalKilledTaskCount += progress.getKilledTaskCount();
+ totalFailedTaskAttemptCount += progress.getFailedTaskAttemptCount();
+ totalKilledTaskAttemptCount += progress.getKilledTaskAttemptCount();
+ }
+ ProgressBuilder dagProgress = new ProgressBuilder();
+ dagProgress.setTotalTaskCount(totalTaskCount);
+ dagProgress.setSucceededTaskCount(totalSucceededTaskCount);
+ dagProgress.setRunningTaskCount(totalRunningTaskCount);
+ dagProgress.setFailedTaskCount(totalFailedTaskCount);
+ dagProgress.setKilledTaskCount(totalKilledTaskCount);
+ dagProgress.setFailedTaskAttemptCount(totalFailedTaskAttemptCount);
+ dagProgress.setKilledTaskAttemptCount(totalKilledTaskAttemptCount);
+ return dagProgress;
+ } finally {
+ readLock.unlock();
+ }
+ }
+
@Override
public VertexStatusBuilder getVertexStatus(String vertexName,
Set<StatusGetOpts> statusOptions) {
@@ -940,18 +975,32 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
finishTime = clock.getTime();
}
+ private Map<String, Integer> constructTaskStats(ProgressBuilder progressBuilder) {
+ Map<String, Integer> taskStats = new HashMap<String, Integer>();
+ taskStats.put(ATSConstants.NUM_COMPLETED_TASKS, progressBuilder.getTotalTaskCount());
+ taskStats.put(ATSConstants.NUM_SUCCEEDED_TASKS, progressBuilder.getSucceededTaskCount());
+ taskStats.put(ATSConstants.NUM_FAILED_TASKS, progressBuilder.getFailedTaskCount());
+ taskStats.put(ATSConstants.NUM_KILLED_TASKS, progressBuilder.getKilledTaskCount());
+ taskStats.put(ATSConstants.NUM_FAILED_TASKS_ATTEMPTS,
+ progressBuilder.getFailedTaskAttemptCount());
+ taskStats.put(ATSConstants.NUM_KILLED_TASKS_ATTEMPTS,
+ progressBuilder.getKilledTaskAttemptCount());
+ return taskStats;
+ }
+
void logJobHistoryFinishedEvent() throws IOException {
this.setFinishTime();
+ Map<String, Integer> taskStats = constructTaskStats(getDAGProgress());
DAGFinishedEvent finishEvt = new DAGFinishedEvent(dagId, startTime,
finishTime, DAGState.SUCCEEDED, "", getAllCounters(),
- this.userName, this.dagName);
+ this.userName, this.dagName, taskStats);
this.appContext.getHistoryHandler().handleCriticalEvent(
new DAGHistoryEvent(dagId, finishEvt));
}
void logJobHistoryInitedEvent() {
DAGInitializedEvent initEvt = new DAGInitializedEvent(this.dagId,
- this.initTime, this.userName, this.dagName);
+ this.initTime, this.userName, this.dagName, this.getVertexNameIDMapping());
this.appContext.getHistoryHandler().handle(
new DAGHistoryEvent(dagId, initEvt));
}
@@ -964,10 +1013,11 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
}
void logJobHistoryUnsuccesfulEvent(DAGState state) throws IOException {
+ Map<String, Integer> taskStats = constructTaskStats(getDAGProgress());
DAGFinishedEvent finishEvt = new DAGFinishedEvent(dagId, startTime,
clock.getTime(), state,
StringUtils.join(getDiagnostics(), LINE_SEPARATOR),
- getAllCounters(), this.userName, this.dagName);
+ getAllCounters(), this.userName, this.dagName, taskStats);
this.appContext.getHistoryHandler().handleCriticalEvent(
new DAGHistoryEvent(dagId, finishEvt));
}
http://git-wip-us.apache.org/repos/asf/tez/blob/ede0e645/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index 5c76a77..4edd12b 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -1520,6 +1520,8 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
taskStats.put(ATSConstants.NUM_SUCCEEDED_TASKS, succeededTaskCount);
taskStats.put(ATSConstants.NUM_FAILED_TASKS, failedTaskCount);
taskStats.put(ATSConstants.NUM_KILLED_TASKS, killedTaskCount);
+ taskStats.put(ATSConstants.NUM_FAILED_TASKS_ATTEMPTS, failedTaskAttemptCount.get());
+ taskStats.put(ATSConstants.NUM_KILLED_TASKS_ATTEMPTS, killedTaskAttemptCount.get());
VertexFinishedEvent finishEvt = new VertexFinishedEvent(vertexId, vertexName, initTimeRequested,
initedTime, startTimeRequested, startedTime, finishTime, finalState, diagnostics,
http://git-wip-us.apache.org/repos/asf/tez/blob/ede0e645/tez-dag/src/main/java/org/apache/tez/dag/history/HistoryEventType.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/HistoryEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/history/HistoryEventType.java
index fd747e0..17df58f 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/HistoryEventType.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/HistoryEventType.java
@@ -19,6 +19,7 @@
package org.apache.tez.dag.history;
public enum HistoryEventType {
+ APP_LAUNCHED,
AM_LAUNCHED,
AM_STARTED,
DAG_SUBMITTED,
http://git-wip-us.apache.org/repos/asf/tez/blob/ede0e645/tez-dag/src/main/java/org/apache/tez/dag/history/events/AppLaunchedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/AppLaunchedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/AppLaunchedEvent.java
new file mode 100644
index 0000000..4c79c53
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/AppLaunchedEvent.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.history.events;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.dag.history.HistoryEvent;
+import org.apache.tez.dag.history.HistoryEventType;
+
+public class AppLaunchedEvent implements HistoryEvent {
+
+ private ApplicationId applicationId;
+ private long launchTime;
+ private long appSubmitTime;
+ private String user;
+ private Configuration conf;
+
+ public AppLaunchedEvent() {
+ }
+
+ public AppLaunchedEvent(ApplicationId appId,
+ long launchTime, long appSubmitTime, String user,
+ Configuration conf) {
+ this.applicationId = appId;
+ this.launchTime = launchTime;
+ this.appSubmitTime = appSubmitTime;
+ this.user = user;
+ this.conf = conf;
+ }
+
+ @Override
+ public HistoryEventType getEventType() {
+ return HistoryEventType.APP_LAUNCHED;
+ }
+
+ @Override
+ public boolean isRecoveryEvent() {
+ return false;
+ }
+
+ @Override
+ public boolean isHistoryEvent() {
+ return true;
+ }
+
+ @Override
+ public void toProtoStream(OutputStream outputStream) throws IOException {
+ throw new UnsupportedOperationException("Not a recovery event");
+ }
+
+ @Override
+ public void fromProtoStream(InputStream inputStream) throws IOException {
+ throw new UnsupportedOperationException("Not a recovery event");
+ }
+
+ @Override
+ public String toString() {
+ return "applicationId=" + applicationId
+ + ", appSubmitTime=" + appSubmitTime
+ + ", launchTime=" + launchTime;
+ }
+
+ public ApplicationId getApplicationId() {
+ return applicationId;
+ }
+
+ public long getLaunchTime() {
+ return launchTime;
+ }
+
+ public long getAppSubmitTime() {
+ return appSubmitTime;
+ }
+
+ public String getUser() {
+ return user;
+ }
+
+ public Configuration getConf() {
+ return conf;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/ede0e645/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
index 21199f4..e05f043 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java
@@ -21,6 +21,7 @@ package org.apache.tez.dag.history.events;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -51,13 +52,15 @@ public class DAGFinishedEvent implements HistoryEvent, SummaryEvent {
private String user;
private String dagName;
+ Map<String, Integer> dagTaskStats;
+
public DAGFinishedEvent() {
}
public DAGFinishedEvent(TezDAGID dagId, long startTime,
long finishTime, DAGState state,
String diagnostics, TezCounters counters,
- String user, String dagName) {
+ String user, String dagName, Map<String, Integer> dagTaskStats) {
this.dagID = dagId;
this.startTime = startTime;
this.finishTime = finishTime;
@@ -66,6 +69,7 @@ public class DAGFinishedEvent implements HistoryEvent, SummaryEvent {
this.tezCounters = counters;
this.user = user;
this.dagName = dagName;
+ this.dagTaskStats = dagTaskStats;
}
@Override
@@ -194,4 +198,9 @@ public class DAGFinishedEvent implements HistoryEvent, SummaryEvent {
public String getDagName() {
return dagName;
}
+
+ public Map<String, Integer> getDagTaskStats() {
+ return dagTaskStats;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/tez/blob/ede0e645/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGInitializedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGInitializedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGInitializedEvent.java
index 6e17da8..98d64d3 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGInitializedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGInitializedEvent.java
@@ -21,10 +21,12 @@ package org.apache.tez.dag.history.events;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.util.Map;
import org.apache.tez.dag.history.HistoryEvent;
import org.apache.tez.dag.history.HistoryEventType;
import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezVertexID;
import org.apache.tez.dag.recovery.records.RecoveryProtos;
public class DAGInitializedEvent implements HistoryEvent {
@@ -33,16 +35,18 @@ public class DAGInitializedEvent implements HistoryEvent {
private long initTime;
private String user;
private String dagName;
+ private Map<String, TezVertexID> vertexNameIDMap;
public DAGInitializedEvent() {
}
public DAGInitializedEvent(TezDAGID dagID, long initTime,
- String user, String dagName) {
+ String user, String dagName, Map<String, TezVertexID> vertexNameIDMap) {
this.dagID = dagID;
this.initTime = initTime;
this.user = user;
this.dagName = dagName;
+ this.vertexNameIDMap = vertexNameIDMap;
}
@Override
@@ -109,4 +113,8 @@ public class DAGInitializedEvent implements HistoryEvent {
return dagName;
}
+ public Map<String, TezVertexID> getVertexNameIDMap() {
+ return vertexNameIDMap;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/tez/blob/ede0e645/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGSubmittedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGSubmittedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGSubmittedEvent.java
index 0074a4e..7f0fab3 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGSubmittedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGSubmittedEvent.java
@@ -54,7 +54,6 @@ public class DAGSubmittedEvent implements HistoryEvent, SummaryEvent {
private ApplicationAttemptId applicationAttemptId;
private String user;
private Map<String, LocalResource> cumulativeAdditionalLocalResources;
- private Map<String, TezVertexID> vertexNameIDMap;
public DAGSubmittedEvent() {
}
@@ -62,7 +61,7 @@ public class DAGSubmittedEvent implements HistoryEvent, SummaryEvent {
public DAGSubmittedEvent(TezDAGID dagID, long submitTime,
DAGProtos.DAGPlan dagPlan, ApplicationAttemptId applicationAttemptId,
Map<String, LocalResource> cumulativeAdditionalLocalResources,
- String user, Map<String, TezVertexID> vertexNameIDMap) {
+ String user) {
this.dagID = dagID;
this.dagName = dagPlan.getName();
this.submitTime = submitTime;
@@ -70,7 +69,6 @@ public class DAGSubmittedEvent implements HistoryEvent, SummaryEvent {
this.applicationAttemptId = applicationAttemptId;
this.cumulativeAdditionalLocalResources = cumulativeAdditionalLocalResources;
this.user = user;
- this.vertexNameIDMap = vertexNameIDMap;
}
@Override
@@ -185,8 +183,4 @@ public class DAGSubmittedEvent implements HistoryEvent, SummaryEvent {
return user;
}
- public Map<String, TezVertexID> getVertexNameIDMap() {
- return vertexNameIDMap;
- }
-
}
http://git-wip-us.apache.org/repos/asf/tez/blob/ede0e645/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java
index 9323270..c367d5c 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java
@@ -56,6 +56,7 @@ public class TaskFinishedEvent implements HistoryEvent {
this.state = state;
this.diagnostics = diagnostics;
this.tezCounters = counters;
+ this.successfulAttemptID = successfulAttemptID;
}
public TaskFinishedEvent() {
http://git-wip-us.apache.org/repos/asf/tez/blob/ede0e645/tez-dag/src/main/java/org/apache/tez/dag/history/logging/EntityTypes.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/logging/EntityTypes.java b/tez-dag/src/main/java/org/apache/tez/dag/history/logging/EntityTypes.java
index 00cac28..e2f0882 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/logging/EntityTypes.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/logging/EntityTypes.java
@@ -19,6 +19,7 @@
package org.apache.tez.dag.history.logging;
public enum EntityTypes {
+ TEZ_APPLICATION,
TEZ_APPLICATION_ATTEMPT,
TEZ_CONTAINER_ID,
TEZ_DAG_ID,
http://git-wip-us.apache.org/repos/asf/tez/blob/ede0e645/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/HistoryEventJsonConversion.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/HistoryEventJsonConversion.java b/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/HistoryEventJsonConversion.java
index ad42392..0b6f9d2 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/HistoryEventJsonConversion.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/HistoryEventJsonConversion.java
@@ -23,6 +23,7 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Map.Entry;
+import java.util.TreeMap;
import org.apache.tez.common.ATSConstants;
import org.apache.tez.dag.api.EdgeManagerPluginDescriptor;
@@ -30,6 +31,7 @@ import org.apache.tez.dag.history.HistoryEvent;
import org.apache.tez.dag.history.HistoryEventType;
import org.apache.tez.dag.history.events.AMLaunchedEvent;
import org.apache.tez.dag.history.events.AMStartedEvent;
+import org.apache.tez.dag.history.events.AppLaunchedEvent;
import org.apache.tez.dag.history.events.ContainerLaunchedEvent;
import org.apache.tez.dag.history.events.ContainerStoppedEvent;
import org.apache.tez.dag.history.events.DAGFinishedEvent;
@@ -60,6 +62,9 @@ public class HistoryEventJsonConversion {
}
JSONObject jsonObject = null;
switch (historyEvent.getEventType()) {
+ case APP_LAUNCHED:
+ jsonObject = convertAppLaunchedEvent((AppLaunchedEvent) historyEvent);
+ break;
case AM_LAUNCHED:
jsonObject = convertAMLaunchedEvent((AMLaunchedEvent) historyEvent);
break;
@@ -122,6 +127,24 @@ public class HistoryEventJsonConversion {
return jsonObject;
}
+ private static JSONObject convertAppLaunchedEvent(AppLaunchedEvent event) throws JSONException {
+ JSONObject jsonObject = new JSONObject();
+ jsonObject.put(ATSConstants.ENTITY,
+ "tez_" + event.getApplicationId().toString());
+ jsonObject.put(ATSConstants.ENTITY_TYPE,
+ EntityTypes.TEZ_APPLICATION.name());
+
+ // Other info to tag with Tez App
+ JSONObject otherInfo = new JSONObject();
+ otherInfo.put(ATSConstants.USER, event.getUser());
+ otherInfo.put(ATSConstants.CONFIG, new JSONObject(
+ DAGUtils.convertConfigurationToATSMap(event.getConf())));
+
+ jsonObject.put(ATSConstants.OTHER_INFO, otherInfo);
+
+ return jsonObject;
+ }
+
private static JSONObject convertAMLaunchedEvent(AMLaunchedEvent event) throws JSONException {
JSONObject jsonObject = new JSONObject();
jsonObject.put(ATSConstants.ENTITY,
@@ -307,6 +330,14 @@ public class HistoryEventJsonConversion {
otherInfo.put(ATSConstants.DIAGNOSTICS, event.getDiagnostics());
otherInfo.put(ATSConstants.COUNTERS,
DAGUtils.convertCountersToJSON(event.getTezCounters()));
+
+ final Map<String, Integer> dagTaskStats = event.getDagTaskStats();
+ if (dagTaskStats != null) {
+ for(Entry<String, Integer> entry : dagTaskStats.entrySet()) {
+ otherInfo.put(entry.getKey(), entry.getValue().intValue());
+ }
+ }
+
jsonObject.put(ATSConstants.OTHER_INFO, otherInfo);
return jsonObject;
@@ -330,6 +361,17 @@ public class HistoryEventJsonConversion {
events.put(initEvent);
jsonObject.put(ATSConstants.EVENTS, events);
+ JSONObject otherInfo = new JSONObject();
+
+ if (event.getVertexNameIDMap() != null) {
+ Map<String, String> nameIdStrMap = new TreeMap<String, String>();
+ for (Entry<String, TezVertexID> entry : event.getVertexNameIDMap().entrySet()) {
+ nameIdStrMap.put(entry.getKey(), entry.getValue().toString());
+ }
+ otherInfo.put(ATSConstants.VERTEX_NAME_ID_MAPPING, nameIdStrMap);
+ }
+ jsonObject.put(ATSConstants.OTHER_INFO, otherInfo);
+
return jsonObject;
}
@@ -411,7 +453,7 @@ public class HistoryEventJsonConversion {
// Other info such as dag plan
JSONObject otherInfo = new JSONObject();
otherInfo.put(ATSConstants.DAG_PLAN,
- DAGUtils.generateSimpleJSONPlan(event.getDAGPlan(), event.getVertexNameIDMap()));
+ DAGUtils.generateSimpleJSONPlan(event.getDAGPlan()));
jsonObject.put(ATSConstants.OTHER_INFO, otherInfo);
return jsonObject;
@@ -510,6 +552,9 @@ public class HistoryEventJsonConversion {
otherInfo.put(ATSConstants.DIAGNOSTICS, event.getDiagnostics());
otherInfo.put(ATSConstants.COUNTERS,
DAGUtils.convertCountersToJSON(event.getTezCounters()));
+ if (event.getSuccessfulAttemptID() != null) {
+ otherInfo.put(ATSConstants.SUCCESSFUL_ATTEMPT_ID, event.getSuccessfulAttemptID().toString());
+ }
jsonObject.put(ATSConstants.OTHER_INFO, otherInfo);
@@ -576,8 +621,8 @@ public class HistoryEventJsonConversion {
final Map<String, Integer> vertexTaskStats = event.getVertexTaskStats();
if (vertexTaskStats != null) {
- for(String key : vertexTaskStats.keySet()) {
- otherInfo.put(key, vertexTaskStats.get(key));
+ for(Entry<String, Integer> entry : vertexTaskStats.entrySet()) {
+ otherInfo.put(entry.getKey(), entry.getValue().intValue());
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/ede0e645/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
index 115f739..0bcbcbe 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
@@ -22,9 +22,13 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;
+import java.util.Map.Entry;
+import java.util.TreeMap;
+import org.apache.hadoop.conf.Configuration;
import org.apache.tez.common.ATSConstants;
import org.apache.tez.common.counters.CounterGroup;
import org.apache.tez.common.counters.TezCounter;
@@ -85,11 +89,10 @@ public class DAGUtils {
- public static JSONObject generateSimpleJSONPlan(DAGPlan dagPlan,
- Map<String, TezVertexID> vertexNameIDMap) throws JSONException {
+ public static JSONObject generateSimpleJSONPlan(DAGPlan dagPlan) throws JSONException {
JSONObject dagJson;
try {
- dagJson = new JSONObject(convertDAGPlanToATSMap(dagPlan, vertexNameIDMap));
+ dagJson = new JSONObject(convertDAGPlanToATSMap(dagPlan));
} catch (IOException e) {
throw new TezUncheckedException(e);
}
@@ -129,8 +132,7 @@ public class DAGUtils {
return object;
}
- public static Map<String,Object> convertDAGPlanToATSMap(
- DAGPlan dagPlan, Map<String, TezVertexID> vertexNameIDMap) throws IOException {
+ public static Map<String,Object> convertDAGPlanToATSMap(DAGPlan dagPlan) throws IOException {
final String VERSION_KEY = "version";
final int version = 1;
@@ -141,12 +143,6 @@ public class DAGUtils {
for (DAGProtos.VertexPlan vertexPlan : dagPlan.getVertexList()) {
Map<String,Object> vertexMap = new LinkedHashMap<String, Object>();
vertexMap.put(VERTEX_NAME_KEY, vertexPlan.getName());
- if (vertexNameIDMap != null && !vertexNameIDMap.isEmpty()) {
- TezVertexID vertexID = vertexNameIDMap.get(vertexPlan.getName());
- if (vertexID != null) {
- vertexMap.put(VERTEX_ID_KEY, vertexID.toString());
- }
- }
if (vertexPlan.hasProcessorDescriptor()) {
vertexMap.put(PROCESSOR_CLASS_KEY,
vertexPlan.getProcessorDescriptor().getClassName());
@@ -361,4 +357,14 @@ public class DAGUtils {
return jsonDescriptor;
}
+ public static Map<String, String> convertConfigurationToATSMap(Configuration conf) {
+ Iterator<Entry<String, String>> iter = conf.iterator();
+ Map<String, String> atsConf = new TreeMap<String, String>();
+ while (iter.hasNext()) {
+ Entry<String, String> entry = iter.next();
+ atsConf.put(entry.getKey(), entry.getValue());
+ }
+ return atsConf;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/tez/blob/ede0e645/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGRecovery.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGRecovery.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGRecovery.java
index f05a330..1edefa7 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGRecovery.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGRecovery.java
@@ -100,7 +100,7 @@ public class TestDAGRecovery {
private void restoreFromDAGInitializedEvent() {
DAGState recoveredState =
dag.restoreFromEvent(new DAGInitializedEvent(dagId, initTime, user,
- dagName));
+ dagName, null));
assertEquals(DAGState.INITED, recoveredState);
assertEquals(initTime, dag.initTime);
assertEquals(6, dag.getVertices().size());
@@ -144,7 +144,7 @@ public class TestDAGRecovery {
private void restoreFromDAGFinishedEvent(DAGState finalState) {
DAGState recoveredState =
dag.restoreFromEvent(new DAGFinishedEvent(dagId, startTime, finishTime,
- finalState, "", tezCounters, user, dagName));
+ finalState, "", tezCounters, user, dagName, null));
assertEquals(finishTime, dag.finishTime);
assertFalse(dag.recoveryCommitInProgress);
assertEquals(finalState, recoveredState);
http://git-wip-us.apache.org/repos/asf/tez/blob/ede0e645/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java b/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java
index ad508b6..a7a23db 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java
@@ -18,9 +18,12 @@
package org.apache.tez.dag.history.events;
+import static org.junit.Assert.fail;
+
import java.nio.ByteBuffer;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
@@ -114,6 +117,20 @@ public class TestHistoryEventsProtoConversion {
LOG.info("Deserialized Event toString: " + deserializedEvent.toString());
}
+ private void testAppLaunchedEvent() throws Exception {
+ AppLaunchedEvent event = new AppLaunchedEvent(ApplicationId.newInstance(0, 1),
+ 100, 100, null, new Configuration(false));
+ try {
+ testProtoConversion(event);
+ fail("Expected to fail on conversion");
+ } catch (UnsupportedOperationException e) {
+ // Expected
+ }
+
+ LOG.info("Initial Event toString: " + event.toString());
+
+ }
+
private void testAMLaunchedEvent() throws Exception {
AMLaunchedEvent event = new AMLaunchedEvent(
ApplicationAttemptId.newInstance(
@@ -148,7 +165,7 @@ public class TestHistoryEventsProtoConversion {
ApplicationId.newInstance(0, 1), 1), 1001l,
DAGPlan.newBuilder().setName("foo").build(),
ApplicationAttemptId.newInstance(
- ApplicationId.newInstance(0, 1), 1), null, "", null);
+ ApplicationId.newInstance(0, 1), 1), null, "");
DAGSubmittedEvent deserializedEvent = (DAGSubmittedEvent)
testProtoConversion(event);
Assert.assertEquals(event.getApplicationAttemptId(),
@@ -167,7 +184,7 @@ public class TestHistoryEventsProtoConversion {
private void testDAGInitializedEvent() throws Exception {
DAGInitializedEvent event = new DAGInitializedEvent(
TezDAGID.getInstance(ApplicationId.newInstance(0, 1), 1), 100334l,
- "user", "dagName");
+ "user", "dagName", null);
DAGInitializedEvent deserializedEvent = (DAGInitializedEvent)
testProtoConversion(event);
Assert.assertEquals(event.getDagID(),
@@ -192,7 +209,7 @@ public class TestHistoryEventsProtoConversion {
{
DAGFinishedEvent event = new DAGFinishedEvent(
TezDAGID.getInstance(ApplicationId.newInstance(0, 1), 1), 1000l, 20000l,
- DAGState.FAILED, null, null, "user", "dagName");
+ DAGState.FAILED, null, null, "user", "dagName", null);
DAGFinishedEvent deserializedEvent = (DAGFinishedEvent)
testProtoConversion(event);
Assert.assertEquals(
@@ -213,7 +230,7 @@ public class TestHistoryEventsProtoConversion {
DAGFinishedEvent event = new DAGFinishedEvent(
TezDAGID.getInstance(ApplicationId.newInstance(0, 1), 1), 1000l, 20000l,
DAGState.FAILED, "bad diagnostics", tezCounters,
- "user", "dagName");
+ "user", "dagName", null);
DAGFinishedEvent deserializedEvent = (DAGFinishedEvent)
testProtoConversion(event);
Assert.assertEquals(
@@ -531,7 +548,7 @@ public class TestHistoryEventsProtoConversion {
event = new VertexRecoverableEventsGeneratedEvent(
TezVertexID.getInstance(
TezDAGID.getInstance(ApplicationId.newInstance(0, 1), 1), 1), null);
- Assert.fail("Invalid creation should have errored out");
+ fail("Invalid creation should have errored out");
} catch (RuntimeException e) {
// Expected
}
@@ -617,6 +634,9 @@ public class TestHistoryEventsProtoConversion {
public void testDefaultProtoConversion() throws Exception {
for (HistoryEventType eventType : HistoryEventType.values()) {
switch (eventType) {
+ case APP_LAUNCHED:
+ testAppLaunchedEvent();
+ break;
case AM_LAUNCHED:
testAMLaunchedEvent();
break;
http://git-wip-us.apache.org/repos/asf/tez/blob/ede0e645/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java b/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java
index c9384e1..c3d51c3 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java
@@ -22,6 +22,7 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Random;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -38,6 +39,7 @@ import org.apache.tez.dag.history.HistoryEvent;
import org.apache.tez.dag.history.HistoryEventType;
import org.apache.tez.dag.history.events.AMLaunchedEvent;
import org.apache.tez.dag.history.events.AMStartedEvent;
+import org.apache.tez.dag.history.events.AppLaunchedEvent;
import org.apache.tez.dag.history.events.ContainerLaunchedEvent;
import org.apache.tez.dag.history.events.ContainerStoppedEvent;
import org.apache.tez.dag.history.events.DAGCommitStartedEvent;
@@ -101,6 +103,10 @@ public class TestHistoryEventJsonConversion {
for (HistoryEventType eventType : HistoryEventType.values()) {
HistoryEvent event = null;
switch (eventType) {
+ case APP_LAUNCHED:
+ event = new AppLaunchedEvent(applicationId, random.nextInt(), random.nextInt(),
+ user, new Configuration(false));
+ break;
case AM_LAUNCHED:
event = new AMLaunchedEvent(applicationAttemptId, random.nextInt(), random.nextInt(),
user);
@@ -110,17 +116,17 @@ public class TestHistoryEventJsonConversion {
break;
case DAG_SUBMITTED:
event = new DAGSubmittedEvent(tezDAGID, random.nextInt(), dagPlan, applicationAttemptId,
- null, user, null);
+ null, user);
break;
case DAG_INITIALIZED:
- event = new DAGInitializedEvent(tezDAGID, random.nextInt(), user, dagPlan.getName());
+ event = new DAGInitializedEvent(tezDAGID, random.nextInt(), user, dagPlan.getName(), null);
break;
case DAG_STARTED:
event = new DAGStartedEvent(tezDAGID, random.nextInt(), user, dagPlan.getName());
break;
case DAG_FINISHED:
event = new DAGFinishedEvent(tezDAGID, random.nextInt(), random.nextInt(), DAGState.ERROR,
- null, null, user, dagPlan.getName());
+ null, null, user, dagPlan.getName(), null);
break;
case VERTEX_INITIALIZED:
event = new VertexInitializedEvent(tezVertexID, "v1", random.nextInt(), random.nextInt(),
http://git-wip-us.apache.org/repos/asf/tez/blob/ede0e645/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java b/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java
index e50c67c..6f4202a 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java
@@ -108,7 +108,7 @@ public class TestDAGUtils {
idNameMap.put("vertex2", vId2);
idNameMap.put("vertex3", vId3);
- Map<String, Object> atsMap = DAGUtils.convertDAGPlanToATSMap(dagPlan, idNameMap);
+ Map<String, Object> atsMap = DAGUtils.convertDAGPlanToATSMap(dagPlan);
Assert.assertTrue(atsMap.containsKey(DAGUtils.DAG_NAME_KEY));
Assert.assertEquals(dagPlan.getName(), atsMap.get(DAGUtils.DAG_NAME_KEY));
Assert.assertTrue(atsMap.containsKey("version"));
@@ -128,10 +128,7 @@ public class TestDAGUtils {
for (Object o : ((Collection<?>) atsMap.get(DAGUtils.VERTICES_KEY))) {
Map<String, Object> v = (Map<String, Object>) o;
Assert.assertTrue(v.containsKey(DAGUtils.VERTEX_NAME_KEY));
- Assert.assertTrue(v.containsKey(DAGUtils.VERTEX_ID_KEY));
- String vId = (String)v.get(DAGUtils.VERTEX_ID_KEY);
String vName = (String)v.get(DAGUtils.VERTEX_NAME_KEY);
- Assert.assertEquals(idNameMap.get(vName).toString(), vId);
Assert.assertTrue(v.containsKey(DAGUtils.PROCESSOR_CLASS_KEY));
Assert.assertTrue(v.containsKey(DAGUtils.USER_PAYLOAD_AS_TEXT));
http://git-wip-us.apache.org/repos/asf/tez/blob/ede0e645/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
index 1d569d6..a492408 100644
--- a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
+++ b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.TreeMap;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
@@ -32,6 +33,7 @@ import org.apache.tez.dag.history.HistoryEvent;
import org.apache.tez.dag.history.HistoryEventType;
import org.apache.tez.dag.history.events.AMLaunchedEvent;
import org.apache.tez.dag.history.events.AMStartedEvent;
+import org.apache.tez.dag.history.events.AppLaunchedEvent;
import org.apache.tez.dag.history.events.ContainerLaunchedEvent;
import org.apache.tez.dag.history.events.ContainerStoppedEvent;
import org.apache.tez.dag.history.events.DAGFinishedEvent;
@@ -48,6 +50,7 @@ import org.apache.tez.dag.history.events.VertexParallelismUpdatedEvent;
import org.apache.tez.dag.history.events.VertexStartedEvent;
import org.apache.tez.dag.history.logging.EntityTypes;
import org.apache.tez.dag.history.utils.DAGUtils;
+import org.apache.tez.dag.records.TezVertexID;
public class HistoryEventTimelineConversion {
@@ -58,6 +61,9 @@ public class HistoryEventTimelineConversion {
}
TimelineEntity timelineEntity = null;
switch (historyEvent.getEventType()) {
+ case APP_LAUNCHED:
+ timelineEntity = convertAppLaunchedEvent((AppLaunchedEvent) historyEvent);
+ break;
case AM_LAUNCHED:
timelineEntity = convertAMLaunchedEvent((AMLaunchedEvent) historyEvent);
break;
@@ -121,6 +127,24 @@ public class HistoryEventTimelineConversion {
return timelineEntity;
}
+ private static TimelineEntity convertAppLaunchedEvent(AppLaunchedEvent event) {
+ TimelineEntity atsEntity = new TimelineEntity();
+ atsEntity.setEntityId("tez_"
+ + event.getApplicationId().toString());
+ atsEntity.setEntityType(EntityTypes.TEZ_APPLICATION.name());
+
+ atsEntity.addRelatedEntity(ATSConstants.APPLICATION_ID,
+ event.getApplicationId().toString());
+ atsEntity.addRelatedEntity(ATSConstants.USER, event.getUser());
+
+ atsEntity.addPrimaryFilter(ATSConstants.USER, event.getUser());
+
+ atsEntity.addOtherInfo(ATSConstants.CONFIG,
+ DAGUtils.convertConfigurationToATSMap(event.getConf()));
+
+ return atsEntity;
+ }
+
private static TimelineEntity convertAMLaunchedEvent(AMLaunchedEvent event) {
TimelineEntity atsEntity = new TimelineEntity();
atsEntity.setEntityId("tez_"
@@ -234,6 +258,13 @@ public class HistoryEventTimelineConversion {
atsEntity.addOtherInfo(ATSConstants.COUNTERS,
DAGUtils.convertCountersToATSMap(event.getTezCounters()));
+ final Map<String, Integer> dagTaskStats = event.getDagTaskStats();
+ if (dagTaskStats != null) {
+ for(Entry<String, Integer> entry : dagTaskStats.entrySet()) {
+ atsEntity.addOtherInfo(entry.getKey(), entry.getValue().intValue());
+ }
+ }
+
return atsEntity;
}
@@ -242,16 +273,24 @@ public class HistoryEventTimelineConversion {
atsEntity.setEntityId(event.getDagID().toString());
atsEntity.setEntityType(EntityTypes.TEZ_DAG_ID.name());
- TimelineEvent finishEvt = new TimelineEvent();
- finishEvt.setEventType(HistoryEventType.DAG_INITIALIZED.name());
- finishEvt.setTimestamp(event.getInitTime());
- atsEntity.addEvent(finishEvt);
+ TimelineEvent initEvt = new TimelineEvent();
+ initEvt.setEventType(HistoryEventType.DAG_INITIALIZED.name());
+ initEvt.setTimestamp(event.getInitTime());
+ atsEntity.addEvent(initEvt);
atsEntity.addPrimaryFilter(ATSConstants.USER, event.getUser());
atsEntity.addPrimaryFilter(ATSConstants.DAG_NAME, event.getDagName());
atsEntity.addOtherInfo(ATSConstants.INIT_TIME, event.getInitTime());
+ if (event.getVertexNameIDMap() != null) {
+ Map<String, String> nameIdStrMap = new TreeMap<String, String>();
+ for (Entry<String, TezVertexID> entry : event.getVertexNameIDMap().entrySet()) {
+ nameIdStrMap.put(entry.getKey(), entry.getValue().toString());
+ }
+ atsEntity.addOtherInfo(ATSConstants.VERTEX_NAME_ID_MAPPING, nameIdStrMap);
+ }
+
return atsEntity;
}
@@ -301,7 +340,7 @@ public class HistoryEventTimelineConversion {
try {
atsEntity.addOtherInfo(ATSConstants.DAG_PLAN,
- DAGUtils.convertDAGPlanToATSMap(event.getDAGPlan(), event.getVertexNameIDMap()));
+ DAGUtils.convertDAGPlanToATSMap(event.getDAGPlan()));
} catch (IOException e) {
throw new TezUncheckedException(e);
}
@@ -396,6 +435,10 @@ public class HistoryEventTimelineConversion {
atsEntity.addOtherInfo(ATSConstants.FINISH_TIME, event.getFinishTime());
atsEntity.addOtherInfo(ATSConstants.TIME_TAKEN, (event.getFinishTime() - event.getStartTime()));
atsEntity.addOtherInfo(ATSConstants.STATUS, event.getState().name());
+ if (event.getSuccessfulAttemptID() != null) {
+ atsEntity.addOtherInfo(ATSConstants.SUCCESSFUL_ATTEMPT_ID,
+ event.getSuccessfulAttemptID().toString());
+ }
atsEntity.addOtherInfo(ATSConstants.DIAGNOSTICS, event.getDiagnostics());
atsEntity.addOtherInfo(ATSConstants.COUNTERS,
@@ -459,8 +502,8 @@ public class HistoryEventTimelineConversion {
final Map<String, Integer> vertexTaskStats = event.getVertexTaskStats();
if (vertexTaskStats != null) {
- for(String key : vertexTaskStats.keySet()) {
- atsEntity.addOtherInfo(key, vertexTaskStats.get(key));
+ for(Entry<String, Integer> entry : vertexTaskStats.entrySet()) {
+ atsEntity.addOtherInfo(entry.getKey(), entry.getValue().intValue());
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/ede0e645/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
index 8f95c1e..ba71d46 100644
--- a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
+++ b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
@@ -22,6 +22,7 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Random;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -35,10 +36,12 @@ import org.apache.tez.dag.api.oldrecords.TaskState;
import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
import org.apache.tez.dag.app.dag.DAGState;
import org.apache.tez.dag.app.dag.VertexState;
+import org.apache.tez.dag.app.dag.impl.VertexStats;
import org.apache.tez.dag.history.HistoryEvent;
import org.apache.tez.dag.history.HistoryEventType;
import org.apache.tez.dag.history.events.AMLaunchedEvent;
import org.apache.tez.dag.history.events.AMStartedEvent;
+import org.apache.tez.dag.history.events.AppLaunchedEvent;
import org.apache.tez.dag.history.events.ContainerLaunchedEvent;
import org.apache.tez.dag.history.events.ContainerStoppedEvent;
import org.apache.tez.dag.history.events.DAGCommitStartedEvent;
@@ -101,6 +104,10 @@ public class TestHistoryEventTimelineConversion {
for (HistoryEventType eventType : HistoryEventType.values()) {
HistoryEvent event = null;
switch (eventType) {
+ case APP_LAUNCHED:
+ event = new AppLaunchedEvent(applicationId, random.nextInt(), random.nextInt(),
+ user, new Configuration(false));
+ break;
case AM_LAUNCHED:
event = new AMLaunchedEvent(applicationAttemptId, random.nextInt(), random.nextInt(),
user);
@@ -110,17 +117,17 @@ public class TestHistoryEventTimelineConversion {
break;
case DAG_SUBMITTED:
event = new DAGSubmittedEvent(tezDAGID, random.nextInt(), dagPlan, applicationAttemptId,
- null, user, null);
+ null, user);
break;
case DAG_INITIALIZED:
- event = new DAGInitializedEvent(tezDAGID, random.nextInt(), user, dagPlan.getName());
+ event = new DAGInitializedEvent(tezDAGID, random.nextInt(), user, dagPlan.getName(), null);
break;
case DAG_STARTED:
event = new DAGStartedEvent(tezDAGID, random.nextInt(), user, dagPlan.getName());
break;
case DAG_FINISHED:
event = new DAGFinishedEvent(tezDAGID, random.nextInt(), random.nextInt(), DAGState.ERROR,
- null, null, user, dagPlan.getName());
+ null, null, user, dagPlan.getName(), null);
break;
case VERTEX_INITIALIZED:
event = new VertexInitializedEvent(tezVertexID, "v1", random.nextInt(), random.nextInt(),
@@ -185,6 +192,41 @@ public class TestHistoryEventTimelineConversion {
}
@Test
+ public void testConvertAppLaunchedEvent() {
+ long launchTime = random.nextLong();
+ long submitTime = random.nextLong();
+ Configuration conf = new Configuration(false);
+ conf.set("foo", "bar");
+ conf.set("applicationId", "1234");
+
+
+ AppLaunchedEvent event = new AppLaunchedEvent(applicationId, launchTime,
+ submitTime, user, conf);
+
+ TimelineEntity timelineEntity = HistoryEventTimelineConversion.convertToTimelineEntity(event);
+
+ Assert.assertEquals(EntityTypes.TEZ_APPLICATION.name(), timelineEntity.getEntityType());
+ Assert.assertEquals("tez_" + applicationId.toString(), timelineEntity.getEntityId());
+
+ Assert.assertEquals(2, timelineEntity.getRelatedEntities().size());
+ Assert.assertTrue(timelineEntity.getRelatedEntities().get(ATSConstants.USER).contains(user));
+ Assert.assertTrue(
+ timelineEntity.getRelatedEntities().get(ATSConstants.APPLICATION_ID).contains(
+ applicationId.toString()));
+
+ Assert.assertEquals(1, timelineEntity.getPrimaryFilters().size());
+ Assert.assertTrue(timelineEntity.getPrimaryFilters().get(ATSConstants.USER).contains(user));
+
+ Assert.assertEquals(1, timelineEntity.getOtherInfo().size());
+ Assert.assertTrue(timelineEntity.getOtherInfo().containsKey(ATSConstants.CONFIG));
+
+ Map<String, String> config =
+ (Map<String, String>)timelineEntity.getOtherInfo().get(ATSConstants.CONFIG);
+ Assert.assertEquals(conf.get("foo"), config.get("foo"));
+ Assert.assertEquals(conf.get("applicationId"), config.get("applicationId"));
+ }
+
+ @Test
public void testConvertContainerLaunchedEvent() {
long launchTime = random.nextLong();
ContainerLaunchedEvent event = new ContainerLaunchedEvent(containerId, launchTime,
@@ -219,7 +261,7 @@ public class TestHistoryEventTimelineConversion {
long submitTime = random.nextLong();
DAGSubmittedEvent event = new DAGSubmittedEvent(tezDAGID, submitTime, dagPlan,
- applicationAttemptId, null, user, null);
+ applicationAttemptId, null, user);
TimelineEntity timelineEntity = HistoryEventTimelineConversion.convertToTimelineEntity(event);
Assert.assertEquals(EntityTypes.TEZ_DAG_ID.name(), timelineEntity.getEntityType());
@@ -263,6 +305,92 @@ public class TestHistoryEventTimelineConversion {
}
@Test
+ public void testConvertDAGInitializedEvent() {
+ long initTime = random.nextLong();
+
+ Map<String,TezVertexID> nameIdMap = new HashMap<String, TezVertexID>();
+ nameIdMap.put("foo", tezVertexID);
+
+ DAGInitializedEvent event = new DAGInitializedEvent(tezDAGID, initTime, "user", "dagName",
+ nameIdMap);
+
+ TimelineEntity timelineEntity = HistoryEventTimelineConversion.convertToTimelineEntity(event);
+ Assert.assertEquals(EntityTypes.TEZ_DAG_ID.name(), timelineEntity.getEntityType());
+ Assert.assertEquals(tezDAGID.toString(), timelineEntity.getEntityId());
+
+ Assert.assertEquals(0, timelineEntity.getRelatedEntities().size());
+
+ Assert.assertEquals(1, timelineEntity.getEvents().size());
+ TimelineEvent timelineEvent = timelineEntity.getEvents().get(0);
+ Assert.assertEquals(HistoryEventType.DAG_INITIALIZED.name(), timelineEvent.getEventType());
+ Assert.assertEquals(initTime, timelineEvent.getTimestamp());
+
+ Assert.assertEquals(2, timelineEntity.getPrimaryFilters().size());
+ Assert.assertTrue(
+ timelineEntity.getPrimaryFilters().get(ATSConstants.DAG_NAME).contains("dagName"));
+ Assert.assertTrue(
+ timelineEntity.getPrimaryFilters().get(ATSConstants.USER).contains(user));
+
+ Assert.assertTrue(timelineEntity.getOtherInfo().containsKey(
+ ATSConstants.VERTEX_NAME_ID_MAPPING));
+ Map<String, String> vIdMap = (Map<String, String>) timelineEntity.getOtherInfo().get(
+ ATSConstants.VERTEX_NAME_ID_MAPPING);
+ Assert.assertEquals(1, vIdMap.size());
+ Assert.assertNotNull(vIdMap.containsKey("foo"));
+ Assert.assertEquals(tezVertexID.toString(), vIdMap.get("foo"));
+
+ }
+
+ @Test
+ public void testConvertDAGFinishedEvent() {
+ long finishTime = random.nextLong();
+ long startTime = random.nextLong();
+ Map<String,Integer> taskStats = new HashMap<String, Integer>();
+ taskStats.put("FOO", 100);
+ taskStats.put("BAR", 200);
+
+ DAGFinishedEvent event = new DAGFinishedEvent(tezDAGID, startTime, finishTime, DAGState.ERROR,
+ "diagnostics", null, user, dagPlan.getName(), taskStats);
+
+ TimelineEntity timelineEntity = HistoryEventTimelineConversion.convertToTimelineEntity(event);
+ Assert.assertEquals(EntityTypes.TEZ_DAG_ID.name(), timelineEntity.getEntityType());
+ Assert.assertEquals(tezDAGID.toString(), timelineEntity.getEntityId());
+
+ Assert.assertEquals(0, timelineEntity.getRelatedEntities().size());
+
+ Assert.assertEquals(1, timelineEntity.getEvents().size());
+ TimelineEvent timelineEvent = timelineEntity.getEvents().get(0);
+ Assert.assertEquals(HistoryEventType.DAG_FINISHED.name(), timelineEvent.getEventType());
+ Assert.assertEquals(finishTime, timelineEvent.getTimestamp());
+
+ Assert.assertEquals(3, timelineEntity.getPrimaryFilters().size());
+ Assert.assertTrue(
+ timelineEntity.getPrimaryFilters().get(ATSConstants.DAG_NAME).contains(dagPlan.getName()));
+ Assert.assertTrue(
+ timelineEntity.getPrimaryFilters().get(ATSConstants.USER).contains(user));
+ Assert.assertTrue(
+ timelineEntity.getPrimaryFilters().get(ATSConstants.STATUS).contains(
+ DAGState.ERROR.name()));
+
+ Assert.assertEquals(startTime,
+ ((Long)timelineEntity.getOtherInfo().get(ATSConstants.START_TIME)).longValue());
+ Assert.assertEquals(finishTime,
+ ((Long)timelineEntity.getOtherInfo().get(ATSConstants.FINISH_TIME)).longValue());
+ Assert.assertEquals(finishTime - startTime,
+ ((Long)timelineEntity.getOtherInfo().get(ATSConstants.TIME_TAKEN)).longValue());
+ Assert.assertTrue(timelineEntity.getOtherInfo().containsKey(ATSConstants.COUNTERS));
+ Assert.assertEquals(DAGState.ERROR.name(),
+ timelineEntity.getOtherInfo().get(ATSConstants.STATUS));
+ Assert.assertEquals("diagnostics",
+ timelineEntity.getOtherInfo().get(ATSConstants.DIAGNOSTICS));
+
+ Assert.assertEquals(100,
+ ((Integer)timelineEntity.getOtherInfo().get("FOO")).intValue());
+ Assert.assertEquals(200,
+ ((Integer)timelineEntity.getOtherInfo().get("BAR")).intValue());
+ }
+
+ @Test
public void testConvertVertexInitializedEvent() {
long initRequestedTime = random.nextLong();
long initedTime = random.nextLong();
@@ -306,6 +434,58 @@ public class TestHistoryEventTimelineConversion {
}
@Test
+ public void testConvertVertexFinishedEvent() {
+ long initRequestedTime = random.nextLong();
+ long initedTime = random.nextLong();
+ long startRequestedTime = random.nextLong();
+ long startTime = random.nextLong();
+ long finishTime = random.nextLong();
+ Map<String,Integer> taskStats = new HashMap<String, Integer>();
+ taskStats.put("FOO", 100);
+ taskStats.put("BAR", 200);
+ VertexStats vertexStats = new VertexStats();
+
+ VertexFinishedEvent event = new VertexFinishedEvent(tezVertexID, "v1", initRequestedTime,
+ initedTime, startRequestedTime, startTime, finishTime, VertexState.ERROR,
+ "diagnostics", null, vertexStats, taskStats);
+
+ TimelineEntity timelineEntity = HistoryEventTimelineConversion.convertToTimelineEntity(event);
+ Assert.assertEquals(EntityTypes.TEZ_VERTEX_ID.name(), timelineEntity.getEntityType());
+ Assert.assertEquals(tezVertexID.toString(), timelineEntity.getEntityId());
+
+ Assert.assertEquals(0, timelineEntity.getRelatedEntities().size());
+
+ Assert.assertEquals(2, timelineEntity.getPrimaryFilters().size());
+ Assert.assertTrue(
+ timelineEntity.getPrimaryFilters().get(EntityTypes.TEZ_DAG_ID.name()).contains(
+ tezDAGID.toString()));
+ Assert.assertTrue(
+ timelineEntity.getPrimaryFilters().get(ATSConstants.STATUS).contains(
+ VertexState.ERROR.name()));
+
+ Assert.assertEquals(1, timelineEntity.getEvents().size());
+ TimelineEvent timelineEvent = timelineEntity.getEvents().get(0);
+ Assert.assertEquals(HistoryEventType.VERTEX_FINISHED.name(), timelineEvent.getEventType());
+ Assert.assertEquals(finishTime, timelineEvent.getTimestamp());
+
+ Assert.assertEquals(finishTime,
+ ((Long)timelineEntity.getOtherInfo().get(ATSConstants.FINISH_TIME)).longValue());
+ Assert.assertEquals(finishTime - startTime,
+ ((Long)timelineEntity.getOtherInfo().get(ATSConstants.TIME_TAKEN)).longValue());
+ Assert.assertEquals(VertexState.ERROR.name(),
+ timelineEntity.getOtherInfo().get(ATSConstants.STATUS));
+ Assert.assertEquals("diagnostics",
+ timelineEntity.getOtherInfo().get(ATSConstants.DIAGNOSTICS));
+
+ Assert.assertTrue(timelineEntity.getOtherInfo().containsKey(ATSConstants.STATS));
+
+ Assert.assertEquals(100,
+ ((Integer)timelineEntity.getOtherInfo().get("FOO")).intValue());
+ Assert.assertEquals(200,
+ ((Integer)timelineEntity.getOtherInfo().get("BAR")).intValue());
+ }
+
+ @Test
public void testConvertTaskStartedEvent() {
long scheduleTime = random.nextLong();
long startTime = random.nextLong();
[49/50] [abbrv] tez git commit: TEZ-1690. TestMultiMRInput tests fail
because of user collisions. (Vasanth kumar RJ via hitesh)
Posted by je...@apache.org.
TEZ-1690. TestMultiMRInput tests fail because of user collisions. (Vasanth kumar RJ via hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/571cea4f
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/571cea4f
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/571cea4f
Branch: refs/heads/TEZ-8
Commit: 571cea4f05caf7cb9677ada0f76f924b288580ce
Parents: dd03a81
Author: Hitesh Shah <hi...@apache.org>
Authored: Fri Nov 14 17:20:39 2014 -0800
Committer: Hitesh Shah <hi...@apache.org>
Committed: Fri Nov 14 17:20:39 2014 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/tez/mapreduce/input/TestMultiMRInput.java | 10 ++++++++--
2 files changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/571cea4f/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index aa74705..bb31375 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -12,6 +12,7 @@ ALL CHANGES:
TEZ-1736. Add support for Inputs/Outputs in runtime-library to generate history text data.
TEZ-1721. Update INSTALL instructions for clarifying tez client jars
compatibility with runtime tarball on HDFS.
+ TEZ-1690. TestMultiMRInput tests fail because of user collisions.
Release 0.5.3: Unreleased
http://git-wip-us.apache.org/repos/asf/tez/blob/571cea4f/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/input/TestMultiMRInput.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/input/TestMultiMRInput.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/input/TestMultiMRInput.java
index 05f6bbc..47ec2b7 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/input/TestMultiMRInput.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/input/TestMultiMRInput.java
@@ -53,6 +53,7 @@ import org.apache.tez.runtime.api.Event;
import org.apache.tez.runtime.api.InputContext;
import org.apache.tez.runtime.api.events.InputDataInformationEvent;
import org.apache.tez.runtime.library.api.KeyValueReader;
+import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Test;
@@ -69,8 +70,8 @@ public class TestMultiMRInput {
defaultConf.set("fs.defaultFS", "file:///");
try {
localFs = FileSystem.getLocal(defaultConf);
- testTmpDir = System.getProperty("test.build.data", "/tmp");
- TEST_ROOT_DIR = new Path(testTmpDir, TestMultiMRInput.class.getSimpleName());
+ testTmpDir = System.getProperty("test.build.data", "target");
+ TEST_ROOT_DIR = new Path(testTmpDir, TestMultiMRInput.class.getSimpleName() + "-tmpDir");
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -265,6 +266,11 @@ public class TestMultiMRInput {
return inputContext;
}
+ @AfterClass
+ public static void cleanUp() throws IOException {
+ localFs.delete(TEST_ROOT_DIR, true);
+ }
+
public static LinkedHashMap<LongWritable, Text> createInputData(FileSystem fs, Path workDir,
JobConf job, String filename,
long startKey,
[04/50] [abbrv] tez git commit: TEZ-1699. Vertex.setParallelism
should throw an exception for invalid invocations (bikas)
Posted by je...@apache.org.
TEZ-1699. Vertex.setParallelism should throw an exception for invalid invocations (bikas)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/06e9f88e
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/06e9f88e
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/06e9f88e
Branch: refs/heads/TEZ-8
Commit: 06e9f88eb207f00ae2f31f46477001726754115c
Parents: 4e69bed
Author: Bikas Saha <bi...@apache.org>
Authored: Wed Oct 29 18:12:41 2014 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Wed Oct 29 18:12:41 2014 -0700
----------------------------------------------------------------------
CHANGES.txt | 7 +++
.../tez/dag/api/VertexManagerPluginContext.java | 3 +-
.../java/org/apache/tez/dag/app/dag/Vertex.java | 2 +-
.../apache/tez/dag/app/dag/impl/VertexImpl.java | 36 ++++++-------
.../tez/dag/app/dag/impl/VertexManager.java | 4 +-
.../tez/dag/app/dag/impl/TestDAGImpl.java | 12 ++---
.../tez/dag/app/dag/impl/TestVertexImpl.java | 56 +++++++++++++++-----
7 files changed, 78 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/06e9f88e/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 933a445..7bc96ce 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -14,6 +14,8 @@ INCOMPATIBLE CHANGES
TEZ-1666. UserPayload should be null if the payload is not specified.
0.5.1 client cannot talk to 0.5.2 AMs (TEZ-1666 and TEZ-1664).
context.getUserPayload can now return null, apps may need to add defensive code.
+ TEZ-1699. Vertex.setParallelism should throw an exception for invalid
+ invocations
ALL CHANGES:
TEZ-1620. Wait for application finish before stopping MiniTezCluster
@@ -78,6 +80,11 @@ ALL CHANGES:
TEZ-1701. ATS fixes to flush all history events and also using batching.
TEZ-792. Default staging path should have user name.
TEZ-1689. addendum - fix unit test failure.
+ TEZ-1666. UserPayload should be null if the payload is not specified.
+ 0.5.1 client cannot talk to 0.5.2 AMs (TEZ-1666 and TEZ-1664).
+ context.getUserPayload can now return null, apps may need to add defensive code.
+ TEZ-1699. Vertex.setParallelism should throw an exception for invalid
+ invocations
Release 0.5.1: 2014-10-02
http://git-wip-us.apache.org/repos/asf/tez/blob/06e9f88e/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java b/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java
index 5bd8768..c1f4bcd 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java
@@ -123,9 +123,8 @@ public interface VertexManagerPluginContext {
* @param sourceEdgeManagers Edge Managers to be updated
* @param rootInputSpecUpdate Updated Root Input specifications, if any.
* If none specified, a default of 1 physical input is used
- * @return true if the operation was allowed.
*/
- public boolean setVertexParallelism(int parallelism,
+ public void setVertexParallelism(int parallelism,
@Nullable VertexLocationHint locationHint,
@Nullable Map<String, EdgeManagerPluginDescriptor> sourceEdgeManagers,
@Nullable Map<String, InputSpecUpdate> rootInputSpecUpdate);
http://git-wip-us.apache.org/repos/asf/tez/blob/06e9f88e/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
index cefd34d..fa1f2c4 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java
@@ -87,7 +87,7 @@ public interface Vertex extends Comparable<Vertex> {
@Nullable
TaskLocationHint getTaskLocationHint(TezTaskID taskID);
- boolean setParallelism(int parallelism, VertexLocationHint vertexLocationHint,
+ void setParallelism(int parallelism, VertexLocationHint vertexLocationHint,
Map<String, EdgeManagerPluginDescriptor> sourceEdgeManagers,
Map<String, InputSpecUpdate> rootInputSpecUpdate) throws AMUserCodeException;
void setVertexLocationHint(VertexLocationHint vertexLocationHint);
http://git-wip-us.apache.org/repos/asf/tez/blob/06e9f88e/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index b8e99d4..5c76a77 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -1198,14 +1198,13 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
}
@Override
- public boolean setParallelism(int parallelism, VertexLocationHint vertexLocationHint,
+ public void setParallelism(int parallelism, VertexLocationHint vertexLocationHint,
Map<String, EdgeManagerPluginDescriptor> sourceEdgeManagers,
Map<String, InputSpecUpdate> rootInputSpecUpdates) throws AMUserCodeException {
- return setParallelism(parallelism, vertexLocationHint, sourceEdgeManagers, rootInputSpecUpdates,
- false);
+ setParallelism(parallelism, vertexLocationHint, sourceEdgeManagers, rootInputSpecUpdates, false);
}
- private boolean setParallelism(int parallelism, VertexLocationHint vertexLocationHint,
+ private void setParallelism(int parallelism, VertexLocationHint vertexLocationHint,
Map<String, EdgeManagerPluginDescriptor> sourceEdgeManagers,
Map<String, InputSpecUpdate> rootInputSpecUpdates,
boolean recovering) throws AMUserCodeException {
@@ -1235,7 +1234,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
LOG.info("Got updated RootInputsSpecs during recovery: " + rootInputSpecUpdates.toString());
this.rootInputSpecs.putAll(rootInputSpecUpdates);
}
- return true;
+ return;
} finally {
writeLock.unlock();
}
@@ -1246,8 +1245,9 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
writeLock.lock();
try {
if (parallelismSet == true) {
- LOG.info("Parallelism can only be set dynamically once per vertex: " + logIdentifier);
- return false;
+ String msg = "Parallelism can only be set dynamically once per vertex: " + logIdentifier;
+ LOG.info(msg);
+ throw new TezUncheckedException(msg);
}
parallelismSet = true;
@@ -1311,9 +1311,9 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
if (parallelism >= numTasks) {
// not that hard to support perhaps. but checking right now since there
// is no use case for it and checking may catch other bugs.
- LOG.warn("Increasing parallelism is not supported, vertexId="
- + logIdentifier);
- return false;
+ String msg = "Increasing parallelism is not supported, vertexId=" + logIdentifier;
+ LOG.warn(msg);
+ throw new TezUncheckedException(msg);
}
if (parallelism == numTasks) {
LOG.info("setParallelism same as current value: " + parallelism +
@@ -1342,10 +1342,10 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
Map.Entry<TezTaskID, Task> entry = iter.next();
Task task = entry.getValue();
if (task.getState() != TaskState.NEW) {
- LOG.warn(
- "All tasks must be in initial state when changing parallelism"
- + " for vertex: " + getVertexId() + " name: " + getName());
- return false;
+ String msg = "All tasks must be in initial state when changing parallelism"
+ + " for vertex: " + getVertexId() + " name: " + getName();
+ LOG.warn(msg);
+ throw new TezUncheckedException(msg);
}
if (i <= parallelism) {
continue;
@@ -1407,8 +1407,6 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
} finally {
writeLock.unlock();
}
-
- return true;
}
public void setVertexLocationHint(VertexLocationHint vertexLocationHint) {
@@ -2532,8 +2530,9 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
}
boolean successSetParallelism ;
try {
- successSetParallelism = vertex.setParallelism(0,
+ vertex.setParallelism(0,
null, vertex.recoveredSourceEdgeManagers, vertex.recoveredRootInputSpecUpdates, true);
+ successSetParallelism = true;
} catch (Exception e) {
successSetParallelism = false;
}
@@ -2589,8 +2588,9 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
break;
}
try {
- successSetParallelism = vertex.setParallelism(0, null, vertex.recoveredSourceEdgeManagers,
+ vertex.setParallelism(0, null, vertex.recoveredSourceEdgeManagers,
vertex.recoveredRootInputSpecUpdates, true);
+ successSetParallelism = true;
} catch (Exception e) {
successSetParallelism = false;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/06e9f88e/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java
index c5fe41f..1bfb0f9 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java
@@ -108,11 +108,11 @@ public class VertexManager {
}
@Override
- public boolean setVertexParallelism(int parallelism, VertexLocationHint vertexLocationHint,
+ public void setVertexParallelism(int parallelism, VertexLocationHint vertexLocationHint,
Map<String, EdgeManagerPluginDescriptor> sourceEdgeManagers,
Map<String, InputSpecUpdate> rootInputSpecUpdate) {
try {
- return managedVertex.setParallelism(parallelism, vertexLocationHint, sourceEdgeManagers,
+ managedVertex.setParallelism(parallelism, vertexLocationHint, sourceEdgeManagers,
rootInputSpecUpdate);
} catch (AMUserCodeException e) {
// workaround: convert it to TezUncheckedException which would be caught in VM
http://git-wip-us.apache.org/repos/asf/tez/blob/06e9f88e/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
index af48c49..e79eeef 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
@@ -889,7 +889,7 @@ public class TestDAGImpl {
}
@SuppressWarnings("unchecked")
- @Test()
+ @Test(timeout = 5000)
public void testEdgeManager_GetNumDestinationTaskPhysicalInputs() {
setupDAGWithCustomEdge(ExceptionLocation.GetNumDestinationTaskPhysicalInputs);
dispatcher.getEventHandler().handle(
@@ -912,7 +912,7 @@ public class TestDAGImpl {
}
@SuppressWarnings("unchecked")
- @Test()
+ @Test(timeout = 5000)
public void testEdgeManager_GetNumSourceTaskPhysicalOutputs() {
setupDAGWithCustomEdge(ExceptionLocation.GetNumSourceTaskPhysicalOutputs);
dispatcher.getEventHandler().handle(
@@ -932,7 +932,7 @@ public class TestDAGImpl {
}
@SuppressWarnings("unchecked")
- @Test()
+ @Test(timeout = 5000)
public void testEdgeManager_RouteDataMovementEventToDestination() {
setupDAGWithCustomEdge(ExceptionLocation.RouteDataMovementEventToDestination);
dispatcher.getEventHandler().handle(
@@ -962,7 +962,7 @@ public class TestDAGImpl {
}
@SuppressWarnings("unchecked")
- @Test()
+ @Test(timeout = 5000)
public void testEdgeManager_RouteInputSourceTaskFailedEventToDestination() {
setupDAGWithCustomEdge(ExceptionLocation.RouteInputSourceTaskFailedEventToDestination);
dispatcher.getEventHandler().handle(
@@ -992,7 +992,7 @@ public class TestDAGImpl {
}
@SuppressWarnings("unchecked")
- @Test()
+ @Test(timeout = 5000)
public void testEdgeManager_GetNumDestinationConsumerTasks() {
setupDAGWithCustomEdge(ExceptionLocation.GetNumDestinationConsumerTasks);
dispatcher.getEventHandler().handle(
@@ -1023,7 +1023,7 @@ public class TestDAGImpl {
}
@SuppressWarnings("unchecked")
- @Test()
+ @Test(timeout = 5000)
public void testEdgeManager_RouteInputErrorEventToSource() {
setupDAGWithCustomEdge(ExceptionLocation.RouteInputErrorEventToSource);
dispatcher.getEventHandler().handle(
http://git-wip-us.apache.org/repos/asf/tez/blob/06e9f88e/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
index 55ee05f..fdf0e07 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
@@ -72,6 +72,7 @@ import org.apache.tez.dag.api.OutputDescriptor;
import org.apache.tez.dag.api.ProcessorDescriptor;
import org.apache.tez.dag.api.RootInputLeafOutput;
import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.api.UserPayload;
import org.apache.tez.dag.api.VertexLocationHint;
import org.apache.tez.dag.api.EdgeProperty.DataMovementType;
@@ -2348,7 +2349,7 @@ public class TestVertexImpl {
Map<String, EdgeManagerPluginDescriptor> edgeManagerDescriptors =
Collections.singletonMap(
v1.getName(), mockEdgeManagerDescriptor);
- assertTrue(v3.setParallelism(1, null, edgeManagerDescriptors, null));
+ v3.setParallelism(1, null, edgeManagerDescriptors, null);
assertTrue(v3.sourceVertices.get(v1).getEdgeManager() instanceof
EdgeManagerForTest);
Assert.assertEquals(1, v3.getTotalTasks());
@@ -2357,7 +2358,46 @@ public class TestVertexImpl {
assertTrue(tasks.keySet().iterator().next().equals(firstTask));
}
+
+ @Test(timeout = 5000)
+ public void testVertexSetParallelismIncreaseException() throws Exception {
+ initAllVertices(VertexState.INITED);
+ VertexImpl v3 = vertices.get("vertex3");
+ Assert.assertEquals(2, v3.getTotalTasks());
+ Map<TezTaskID, Task> tasks = v3.getTasks();
+ Assert.assertEquals(2, tasks.size());
+
+ VertexImpl v1 = vertices.get("vertex1");
+ startVertex(vertices.get("vertex2"));
+ startVertex(v1);
+
+ // increase not supported
+ try {
+ v3.setParallelism(100, null, null, null);
+ Assert.fail();
+ } catch (TezUncheckedException e) { }
+ }
+ @Test(timeout = 5000)
+ public void testVertexSetParallelismMultipleException() throws Exception {
+ initAllVertices(VertexState.INITED);
+ VertexImpl v3 = vertices.get("vertex3");
+ Assert.assertEquals(2, v3.getTotalTasks());
+ Map<TezTaskID, Task> tasks = v3.getTasks();
+ Assert.assertEquals(2, tasks.size());
+
+ VertexImpl v1 = vertices.get("vertex1");
+ startVertex(vertices.get("vertex2"));
+ startVertex(v1);
+ v3.setParallelism(1, null, null, null);
+
+ // multiple invocations not supported
+ try {
+ v3.setParallelism(1, null, null, null);
+ Assert.fail();
+ } catch (TezUncheckedException e) { }
+ }
+
@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testVertexPendingTaskEvents() {
@@ -2415,8 +2455,7 @@ public class TestVertexImpl {
Map<String, EdgeManagerPluginDescriptor> edgeManagerDescriptors =
Collections.singletonMap(v3.getName(), edgeManagerDescriptor);
- assertTrue(v5.setParallelism(v5.getTotalTasks() - 1, null,
- edgeManagerDescriptors, null)); // Must decrease.
+ v5.setParallelism(v5.getTotalTasks() - 1, null, edgeManagerDescriptors, null); // Must decrease.
VertexImpl v5Impl = (VertexImpl) v5;
@@ -2938,11 +2977,6 @@ public class TestVertexImpl {
}
- @Test(timeout = 5000)
- public void testCommitterInitAndSetup() {
- // FIXME need to add a test for this
- }
-
@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testBadCommitter() throws Exception {
@@ -3252,10 +3286,6 @@ public class TestVertexImpl {
Assert.assertEquals(VertexState.RUNNING, vertices.get("vertex4").getState());
}
- @Test(timeout = 5000)
- public void testHistoryEventGeneration() {
- }
-
@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testInvalidEvent() {
@@ -3391,7 +3421,7 @@ public class TestVertexImpl {
initializer.stateUpdates.get(1).getVertexState());
}
- @Test(timeout = 1000000)
+ @Test(timeout = 10000)
public void testInputInitializerEventMultipleAttempts() throws Exception {
useCustomInitializer = true;
customInitializer = new EventHandlingRootInputInitializer(null);
[09/50] [abbrv] tez git commit: TEZ-1711. Don't cache outputSpecList
in VertexImpl.getOutputSpecList(taskIndex). (zjffdu)
Posted by je...@apache.org.
TEZ-1711. Don't cache outputSpecList in VertexImpl.getOutputSpecList(taskIndex). (zjffdu)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/7f8fc753
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/7f8fc753
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/7f8fc753
Branch: refs/heads/TEZ-8
Commit: 7f8fc7530b39106e238c67b7bd22d5ed5670d449
Parents: c570a0e
Author: Jeff Zhang <zj...@apache.org>
Authored: Fri Oct 31 08:47:18 2014 +0800
Committer: Jeff Zhang <zj...@apache.org>
Committed: Fri Oct 31 08:47:18 2014 +0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/tez/dag/app/dag/impl/VertexImpl.java | 18 +++++++-----------
.../apache/tez/dag/app/dag/impl/TestDAGImpl.java | 5 ++++-
3 files changed, 12 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/7f8fc753/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 68d4838..0b2dc65 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -91,6 +91,7 @@ ALL CHANGES:
based affinity
TEZ-1716. Additional ATS data for UI.
TEZ-1722. DAG should be related to Application Id in ATS data.
+ TEZ-1711. Don't cache outputSpecList in VertexImpl.getOutputSpecList(taskIndex)
Release 0.5.1: 2014-10-02
http://git-wip-us.apache.org/repos/asf/tez/blob/7f8fc753/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index 4edd12b..c182810 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -224,8 +224,6 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
@VisibleForTesting
int numSuccessSourceAttemptCompletions = 0;
- List<InputSpec> inputSpecList;
- List<OutputSpec> outputSpecList;
List<GroupInputSpec> groupInputSpecList;
Set<String> sharedOutputs = Sets.newHashSet();
@@ -3931,7 +3929,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
// TODO Eventually remove synchronization.
@Override
public synchronized List<InputSpec> getInputSpecList(int taskIndex) throws AMUserCodeException {
- inputSpecList = new ArrayList<InputSpec>(this.getInputVerticesCount()
+ List<InputSpec> inputSpecList = new ArrayList<InputSpec>(this.getInputVerticesCount()
+ (rootInputDescriptors == null ? 0 : rootInputDescriptors.size()));
if (rootInputDescriptors != null) {
for (Entry<String, RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>>
@@ -3956,14 +3954,12 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
// TODO Eventually remove synchronization.
@Override
public synchronized List<OutputSpec> getOutputSpecList(int taskIndex) throws AMUserCodeException {
- if (this.outputSpecList == null) {
- outputSpecList = new ArrayList<OutputSpec>(this.getOutputVerticesCount()
- + this.additionalOutputSpecs.size());
- outputSpecList.addAll(additionalOutputSpecs);
- for (Entry<Vertex, Edge> entry : this.getOutputVertices().entrySet()) {
- OutputSpec outputSpec = entry.getValue().getSourceSpec(taskIndex);
- outputSpecList.add(outputSpec);
- }
+ List<OutputSpec> outputSpecList = new ArrayList<OutputSpec>(this.getOutputVerticesCount()
+ + this.additionalOutputSpecs.size());
+ outputSpecList.addAll(additionalOutputSpecs);
+ for (Entry<Vertex, Edge> entry : this.getOutputVertices().entrySet()) {
+ OutputSpec outputSpec = entry.getValue().getSourceSpec(taskIndex);
+ outputSpecList.add(outputSpec);
}
return outputSpecList;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/7f8fc753/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
index e79eeef..f1961aa 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
@@ -644,6 +644,7 @@ public class TestDAGImpl {
return dag;
}
+ // v1 -> v2
private DAGPlan createDAGWithCustomEdge(ExceptionLocation exLocation) {
LOG.info("Setting up dag plan");
DAGPlan dag = DAGPlan.newBuilder()
@@ -920,7 +921,9 @@ public class TestDAGImpl {
dispatcher.getEventHandler().handle(new DAGEventStartDag(dagWithCustomEdge.getID(),
null));
dispatcher.await();
- Assert.assertEquals(DAGState.RUNNING, dagWithCustomEdge.getState());
+ // After TEZ-1711, all task attempts of v1 fail which result in task fail, and finally
+ // dag failed.
+ Assert.assertEquals(DAGState.FAILED, dagWithCustomEdge.getState());
VertexImpl v1 = (VertexImpl)dagWithCustomEdge.getVertex("vertex1");
Task t1= v1.getTask(0);
[47/50] [abbrv] tez git commit: TEZ-1772. Failing tests post
TEZ-1737. (Jeff Zhang via hitesh)
Posted by je...@apache.org.
TEZ-1772. Failing tests post TEZ-1737. (Jeff Zhang via hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/1a3059f4
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/1a3059f4
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/1a3059f4
Branch: refs/heads/TEZ-8
Commit: 1a3059f41820ce91acc9eeb3b8c30fb4b53ea2b5
Parents: f4c2112
Author: Hitesh Shah <hi...@apache.org>
Authored: Thu Nov 13 14:12:51 2014 -0800
Committer: Hitesh Shah <hi...@apache.org>
Committed: Thu Nov 13 14:12:51 2014 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/tez/dag/history/events/VertexFinishedEvent.java | 2 ++
tez-dag/src/main/proto/HistoryEvents.proto | 1 +
3 files changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/1a3059f4/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 4f0417e..0f66807 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -26,6 +26,7 @@ ALL CHANGES:
TEZ-1737. Should add taskNum in VertexFinishedEvent.
TEZ-1770. Handle ConnectExceptions correctly when establishing connections to an NM which may be down.
TEZ-1774. AppLaunched event for Timeline does not have start time set.
+ TEZ-1772. Failing tests post TEZ-1737.
Release 0.5.2: 2014-11-07
http://git-wip-us.apache.org/repos/asf/tez/blob/1a3059f4/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
index d1d7eef..d6e0b79 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java
@@ -98,6 +98,7 @@ public class VertexFinishedEvent implements HistoryEvent, SummaryEvent {
VertexFinishedProto.Builder builder = VertexFinishedProto.newBuilder();
builder.setVertexName(vertexName)
.setVertexId(vertexID.toString())
+ .setNumTasks(numTasks)
.setState(state.ordinal())
.setFinishTime(finishTime);
if (diagnostics != null) {
@@ -109,6 +110,7 @@ public class VertexFinishedEvent implements HistoryEvent, SummaryEvent {
public void fromProto(VertexFinishedProto proto) {
this.vertexName = proto.getVertexName();
this.vertexID = TezVertexID.fromString(proto.getVertexId());
+ this.numTasks = proto.getNumTasks();
this.finishTime = proto.getFinishTime();
this.state = VertexState.values()[proto.getState()];
if (proto.hasDiagnostics()) {
http://git-wip-us.apache.org/repos/asf/tez/blob/1a3059f4/tez-dag/src/main/proto/HistoryEvents.proto
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/proto/HistoryEvents.proto b/tez-dag/src/main/proto/HistoryEvents.proto
index 5d18b0a..e8f323d 100644
--- a/tez-dag/src/main/proto/HistoryEvents.proto
+++ b/tez-dag/src/main/proto/HistoryEvents.proto
@@ -136,6 +136,7 @@ message VertexFinishedProto {
optional int64 finish_time = 3;
optional int32 state = 4;
optional string diagnostics = 5;
+ optional int32 num_tasks = 6;
}
message TaskStartedProto {
[03/50] [abbrv] tez git commit: TEZ-1666. UserPayload should be null
if the payload is not specified. (sseth)
Posted by je...@apache.org.
TEZ-1666. UserPayload should be null if the payload is not specified.
(sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/4e69bed5
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/4e69bed5
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/4e69bed5
Branch: refs/heads/TEZ-8
Commit: 4e69bed5c7dbc68d17f3d4648838e5acfc52c3eb
Parents: 1ffbc19
Author: Siddharth Seth <ss...@apache.org>
Authored: Wed Oct 29 14:22:03 2014 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Wed Oct 29 14:22:03 2014 -0700
----------------------------------------------------------------------
CHANGES.txt | 3 +
.../apache/tez/dag/api/DagTypeConverters.java | 56 ++--
tez-api/src/main/proto/DAGApiRecords.proto | 8 +-
.../org/apache/tez/dag/api/TestDAGPlan.java | 18 +-
.../tez/dag/api/TestDagTypeConverters.java | 4 +-
.../org/apache/tez/dag/app/dag/impl/Edge.java | 11 +-
.../apache/tez/dag/app/dag/impl/VertexImpl.java | 10 +-
.../tez/dag/app/dag/impl/TestDAGImpl.java | 117 ++++----
.../tez/dag/app/dag/impl/TestVertexImpl.java | 268 ++++++++++---------
.../TestHistoryEventsProtoConversion.java | 3 +-
.../runtime/api/impl/TezInputContextImpl.java | 2 +-
.../api/impl/TezMergedInputContextImpl.java | 2 +-
.../runtime/api/impl/TezOutputContextImpl.java | 2 +-
.../api/impl/TezProcessorContextImpl.java | 2 +-
.../examples/BroadcastAndOneToOneExample.java | 3 +-
15 files changed, 275 insertions(+), 234 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/4e69bed5/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 0d2ccb9..933a445 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -11,6 +11,9 @@ ALL CHANGES:
Release 0.5.2: Unreleased
INCOMPATIBLE CHANGES
+ TEZ-1666. UserPayload should be null if the payload is not specified.
+ 0.5.1 client cannot talk to 0.5.2 AMs (TEZ-1666 and TEZ-1664).
+ context.getUserPayload can now return null, apps may need to add defensive code.
ALL CHANGES:
TEZ-1620. Wait for application finish before stopping MiniTezCluster
http://git-wip-us.apache.org/repos/asf/tez/blob/4e69bed5/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java b/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
index 179f3cc..17807d3 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
@@ -306,9 +306,13 @@ public class DagTypeConverters {
builder.setClassName(descriptor.getClassName());
UserPayload userPayload = descriptor.getUserPayload();
- if (userPayload != null && userPayload.hasPayload()) {
- builder.setUserPayload(ByteString.copyFrom(descriptor.getUserPayload().getPayload()));
- builder.setVersion(userPayload.getVersion());
+ if (userPayload != null) {
+ DAGProtos.TezUserPayloadProto.Builder payloadBuilder = DAGProtos.TezUserPayloadProto.newBuilder();
+ if (userPayload.hasPayload()) {
+ payloadBuilder.setUserPayload(ByteString.copyFrom(userPayload.getPayload()));
+ payloadBuilder.setVersion(userPayload.getVersion());
+ }
+ builder.setTezUserPayload(payloadBuilder.build());
}
if (descriptor.getHistoryText() != null) {
try {
@@ -348,62 +352,84 @@ public class DagTypeConverters {
private static UserPayload convertTezUserPayloadFromDAGPlan(
TezEntityDescriptorProto proto) {
UserPayload userPayload = null;
- if (proto.hasUserPayload()) {
- userPayload =
- UserPayload.create(proto.getUserPayload().asReadOnlyByteBuffer(), proto.getVersion());
- } else {
- userPayload = UserPayload.create(null, -1);
+ if (proto.hasTezUserPayload()) {
+ if (proto.getTezUserPayload().hasUserPayload()) {
+ userPayload =
+ UserPayload.create(proto.getTezUserPayload().getUserPayload().asReadOnlyByteBuffer(), proto.getTezUserPayload().getVersion());
+ } else {
+ userPayload = UserPayload.create(null);
+ }
}
return userPayload;
}
+ private static void setUserPayload(EntityDescriptor<?> entity, UserPayload payload) {
+ if (payload != null) {
+ entity.setUserPayload(payload);
+ }
+ }
+
public static InputDescriptor convertInputDescriptorFromDAGPlan(
TezEntityDescriptorProto proto) {
String className = proto.getClassName();
UserPayload payload = convertTezUserPayloadFromDAGPlan(proto);
- return InputDescriptor.create(className).setUserPayload(payload);
+ InputDescriptor id = InputDescriptor.create(className);
+ setUserPayload(id, payload);
+ return id;
}
public static OutputDescriptor convertOutputDescriptorFromDAGPlan(
TezEntityDescriptorProto proto) {
String className = proto.getClassName();
UserPayload payload = convertTezUserPayloadFromDAGPlan(proto);
- return OutputDescriptor.create(className).setUserPayload(payload);
+ OutputDescriptor od = OutputDescriptor.create(className);
+ setUserPayload(od, payload);
+ return od;
}
public static InputInitializerDescriptor convertInputInitializerDescriptorFromDAGPlan(
TezEntityDescriptorProto proto) {
String className = proto.getClassName();
UserPayload payload = convertTezUserPayloadFromDAGPlan(proto);
- return InputInitializerDescriptor.create(className).setUserPayload(payload);
+ InputInitializerDescriptor iid = InputInitializerDescriptor.create(className);
+ setUserPayload(iid, payload);
+ return iid;
}
public static OutputCommitterDescriptor convertOutputCommitterDescriptorFromDAGPlan(
TezEntityDescriptorProto proto) {
String className = proto.getClassName();
UserPayload payload = convertTezUserPayloadFromDAGPlan(proto);
- return OutputCommitterDescriptor.create(className).setUserPayload(payload);
+ OutputCommitterDescriptor ocd = OutputCommitterDescriptor.create(className);
+ setUserPayload(ocd, payload);
+ return ocd;
}
public static VertexManagerPluginDescriptor convertVertexManagerPluginDescriptorFromDAGPlan(
TezEntityDescriptorProto proto) {
String className = proto.getClassName();
UserPayload payload = convertTezUserPayloadFromDAGPlan(proto);
- return VertexManagerPluginDescriptor.create(className).setUserPayload(payload);
+ VertexManagerPluginDescriptor vmpd = VertexManagerPluginDescriptor.create(className);
+ setUserPayload(vmpd, payload);
+ return vmpd;
}
public static EdgeManagerPluginDescriptor convertEdgeManagerPluginDescriptorFromDAGPlan(
TezEntityDescriptorProto proto) {
String className = proto.getClassName();
UserPayload payload = convertTezUserPayloadFromDAGPlan(proto);
- return EdgeManagerPluginDescriptor.create(className).setUserPayload(payload);
+ EdgeManagerPluginDescriptor empd = EdgeManagerPluginDescriptor.create(className);
+ setUserPayload(empd, payload);
+ return empd;
}
public static ProcessorDescriptor convertProcessorDescriptorFromDAGPlan(
TezEntityDescriptorProto proto) {
String className = proto.getClassName();
UserPayload payload = convertTezUserPayloadFromDAGPlan(proto);
- return ProcessorDescriptor.create(className).setUserPayload(payload);
+ ProcessorDescriptor pd = ProcessorDescriptor.create(className);
+ setUserPayload(pd, payload);
+ return pd;
}
public static TezAppMasterStatus convertTezSessionStatusFromProto(
http://git-wip-us.apache.org/repos/asf/tez/blob/4e69bed5/tez-api/src/main/proto/DAGApiRecords.proto
----------------------------------------------------------------------
diff --git a/tez-api/src/main/proto/DAGApiRecords.proto b/tez-api/src/main/proto/DAGApiRecords.proto
index 04aa791..177faba 100644
--- a/tez-api/src/main/proto/DAGApiRecords.proto
+++ b/tez-api/src/main/proto/DAGApiRecords.proto
@@ -112,9 +112,13 @@ message PlanTaskConfiguration {
message TezEntityDescriptorProto {
optional string class_name = 1;
- optional bytes user_payload = 2;
+ optional TezUserPayloadProto tez_user_payload = 2;
optional bytes history_text = 3;
- optional int32 version = 4;
+}
+
+message TezUserPayloadProto {
+ optional bytes user_payload = 1;
+ optional int32 version = 2;
}
message RootInputLeafOutputProto {
http://git-wip-us.apache.org/repos/asf/tez/blob/4e69bed5/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java b/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
index 8cbd611..fccbb08 100644
--- a/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
+++ b/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
@@ -167,19 +167,19 @@ public class TestDAGPlan {
VertexPlan v2Proto = dagProto.getVertex(1);
EdgePlan edgeProto = dagProto.getEdge(0);
- assertEquals("processor1Bytes", new String(v1Proto.getProcessorDescriptor()
+ assertEquals("processor1Bytes", new String(v1Proto.getProcessorDescriptor().getTezUserPayload()
.getUserPayload().toByteArray()));
assertEquals("processor1", v1Proto.getProcessorDescriptor().getClassName());
- assertEquals("processor2Bytes", new String(v2Proto.getProcessorDescriptor()
+ assertEquals("processor2Bytes", new String(v2Proto.getProcessorDescriptor().getTezUserPayload()
.getUserPayload().toByteArray()));
assertEquals("processor2", v2Proto.getProcessorDescriptor().getClassName());
- assertEquals("inputBytes", new String(edgeProto.getEdgeDestination()
+ assertEquals("inputBytes", new String(edgeProto.getEdgeDestination().getTezUserPayload()
.getUserPayload().toByteArray()));
assertEquals("input", edgeProto.getEdgeDestination().getClassName());
- assertEquals("outputBytes", new String(edgeProto.getEdgeSource()
+ assertEquals("outputBytes", new String(edgeProto.getEdgeSource().getTezUserPayload()
.getUserPayload().toByteArray()));
assertEquals("output", edgeProto.getEdgeSource().getClassName());
@@ -235,8 +235,8 @@ public class TestDAGPlan {
EdgePlan edgeProto = dagProto.getEdge(0);
// either v1 or v2 will be on top based on topological order
- String v1ProtoPayload = new String(v1Proto.getProcessorDescriptor().getUserPayload().toByteArray());
- String v2ProtoPayload = new String(v2Proto.getProcessorDescriptor().getUserPayload().toByteArray());
+ String v1ProtoPayload = new String(v1Proto.getProcessorDescriptor().getTezUserPayload().getUserPayload().toByteArray());
+ String v2ProtoPayload = new String(v2Proto.getProcessorDescriptor().getTezUserPayload().getUserPayload().toByteArray());
assertTrue(v1ProtoPayload.equals("processor1Bytes") || v1ProtoPayload.equals("processor3Bytes"));
assertTrue(v2ProtoPayload.equals("processor1Bytes") || v2ProtoPayload.equals("processor3Bytes"));
assertTrue(v1Proto.getProcessorDescriptor().getClassName().equals("processor1") ||
@@ -244,15 +244,15 @@ public class TestDAGPlan {
assertTrue(v2Proto.getProcessorDescriptor().getClassName().equals("processor1") ||
v2Proto.getProcessorDescriptor().getClassName().equals("processor3"));
- assertEquals("processor2Bytes", new String(v3Proto.getProcessorDescriptor()
+ assertEquals("processor2Bytes", new String(v3Proto.getProcessorDescriptor().getTezUserPayload()
.getUserPayload().toByteArray()));
assertEquals("processor2", v3Proto.getProcessorDescriptor().getClassName());
- assertEquals("inputBytes", new String(edgeProto.getEdgeDestination()
+ assertEquals("inputBytes", new String(edgeProto.getEdgeDestination().getTezUserPayload()
.getUserPayload().toByteArray()));
assertEquals("input", edgeProto.getEdgeDestination().getClassName());
- assertEquals("outputBytes", new String(edgeProto.getEdgeSource()
+ assertEquals("outputBytes", new String(edgeProto.getEdgeSource().getTezUserPayload()
.getUserPayload().toByteArray()));
assertEquals("output", edgeProto.getEdgeSource().getClassName());
http://git-wip-us.apache.org/repos/asf/tez/blob/4e69bed5/tez-api/src/test/java/org/apache/tez/dag/api/TestDagTypeConverters.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/TestDagTypeConverters.java b/tez-api/src/test/java/org/apache/tez/dag/api/TestDagTypeConverters.java
index 9a1cb07..13347bb 100644
--- a/tez-api/src/test/java/org/apache/tez/dag/api/TestDagTypeConverters.java
+++ b/tez-api/src/test/java/org/apache/tez/dag/api/TestDagTypeConverters.java
@@ -37,8 +37,8 @@ public class TestDagTypeConverters {
.setHistoryText(historytext);
TezEntityDescriptorProto proto =
DagTypeConverters.convertToDAGPlan(entityDescriptor);
- Assert.assertEquals(payload.getVersion(), proto.getVersion());
- Assert.assertArrayEquals(payload.deepCopyAsArray(), proto.getUserPayload().toByteArray());
+ Assert.assertEquals(payload.getVersion(), proto.getTezUserPayload().getVersion());
+ Assert.assertArrayEquals(payload.deepCopyAsArray(), proto.getTezUserPayload().getUserPayload().toByteArray());
Assert.assertTrue(proto.hasHistoryText());
Assert.assertNotEquals(historytext, proto.getHistoryText());
Assert.assertEquals(historytext, new String(
http://git-wip-us.apache.org/repos/asf/tez/blob/4e69bed5/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/Edge.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/Edge.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/Edge.java
index 57d742f..360a839 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/Edge.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/Edge.java
@@ -115,25 +115,22 @@ public class Edge {
private void createEdgeManager() {
switch (edgeProperty.getDataMovementType()) {
case ONE_TO_ONE:
- edgeManagerContext = new EdgeManagerPluginContextImpl(UserPayload.create(null));
+ edgeManagerContext = new EdgeManagerPluginContextImpl(null);
edgeManager = new OneToOneEdgeManager(edgeManagerContext);
break;
case BROADCAST:
- edgeManagerContext = new EdgeManagerPluginContextImpl(UserPayload.create(null));
+ edgeManagerContext = new EdgeManagerPluginContextImpl(null);
edgeManager = new BroadcastEdgeManager(edgeManagerContext);
break;
case SCATTER_GATHER:
- edgeManagerContext = new EdgeManagerPluginContextImpl(UserPayload.create(null));
+ edgeManagerContext = new EdgeManagerPluginContextImpl(null);
edgeManager = new ScatterGatherEdgeManager(edgeManagerContext);
break;
case CUSTOM:
if (edgeProperty.getEdgeManagerDescriptor() != null) {
UserPayload payload = null;
- if (edgeProperty.getEdgeManagerDescriptor().getUserPayload() != null &&
- edgeProperty.getEdgeManagerDescriptor().getUserPayload().hasPayload()) {
+ if (edgeProperty.getEdgeManagerDescriptor().getUserPayload() != null) {
payload = edgeProperty.getEdgeManagerDescriptor().getUserPayload();
- } else {
- payload = UserPayload.create(null);
}
edgeManagerContext = new EdgeManagerPluginContextImpl(payload);
String edgeManagerClassName = edgeProperty.getEdgeManagerDescriptor().getClassName();
http://git-wip-us.apache.org/repos/asf/tez/blob/4e69bed5/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index 50e8b0c..b8e99d4 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -69,7 +69,6 @@ import org.apache.tez.dag.api.OutputDescriptor;
import org.apache.tez.dag.api.ProcessorDescriptor;
import org.apache.tez.dag.api.RootInputLeafOutput;
import org.apache.tez.dag.api.TezUncheckedException;
-import org.apache.tez.dag.api.UserPayload;
import org.apache.tez.dag.api.VertexLocationHint;
import org.apache.tez.dag.api.TaskLocationHint;
import org.apache.tez.dag.api.VertexManagerPluginContext.TaskWithLocationHint;
@@ -2032,15 +2031,13 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
LOG.info("Setting vertexManager to RootInputVertexManager for "
+ logIdentifier);
vertexManager = new VertexManager(
- VertexManagerPluginDescriptor.create(RootInputVertexManager.class.getName())
- .setUserPayload(UserPayload.create(null)),
+ VertexManagerPluginDescriptor.create(RootInputVertexManager.class.getName()),
this, appContext);
} else if (hasOneToOne && !hasCustom) {
LOG.info("Setting vertexManager to InputReadyVertexManager for "
+ logIdentifier);
vertexManager = new VertexManager(
- VertexManagerPluginDescriptor.create(InputReadyVertexManager.class.getName())
- .setUserPayload(UserPayload.create(null)),
+ VertexManagerPluginDescriptor.create(InputReadyVertexManager.class.getName()),
this, appContext);
} else if (hasBipartite && !hasCustom) {
LOG.info("Setting vertexManager to ShuffleVertexManager for "
@@ -2053,8 +2050,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
LOG.info("Setting vertexManager to ImmediateStartVertexManager for "
+ logIdentifier);
vertexManager = new VertexManager(
- VertexManagerPluginDescriptor.create(ImmediateStartVertexManager.class.getName())
- .setUserPayload(UserPayload.create(null)),
+ VertexManagerPluginDescriptor.create(ImmediateStartVertexManager.class.getName()),
this, appContext);
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/4e69bed5/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
index dc90a6f..af48c49 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
@@ -650,65 +650,67 @@ public class TestDAGImpl {
.setName("testverteximpl")
.addVertex(
VertexPlan.newBuilder()
- .setName("vertex1")
- .setType(PlanVertexType.NORMAL)
- .addTaskLocationHint(
- PlanTaskLocationHint.newBuilder()
- .addHost("host1")
- .addRack("rack1")
- .build()
+ .setName("vertex1")
+ .setType(PlanVertexType.NORMAL)
+ .addTaskLocationHint(
+ PlanTaskLocationHint.newBuilder()
+ .addHost("host1")
+ .addRack("rack1")
+ .build()
)
- .setTaskConfig(
- PlanTaskConfiguration.newBuilder()
- .setNumTasks(1)
- .setVirtualCores(4)
- .setMemoryMb(1024)
- .setJavaOpts("")
- .setTaskModule("x1.y1")
- .build()
- )
- .addOutEdgeId("e1")
- .build()
- )
- .addVertex(
- VertexPlan.newBuilder()
- .setName("vertex2")
- .setType(PlanVertexType.NORMAL)
- .setProcessorDescriptor(TezEntityDescriptorProto.newBuilder().setClassName("x2.y2"))
- .addTaskLocationHint(
- PlanTaskLocationHint.newBuilder()
- .addHost("host2")
- .addRack("rack2")
- .build()
+ .setTaskConfig(
+ PlanTaskConfiguration.newBuilder()
+ .setNumTasks(1)
+ .setVirtualCores(4)
+ .setMemoryMb(1024)
+ .setJavaOpts("")
+ .setTaskModule("x1.y1")
+ .build()
)
- .setTaskConfig(
- PlanTaskConfiguration.newBuilder()
- .setNumTasks(2)
- .setVirtualCores(4)
- .setMemoryMb(1024)
- .setJavaOpts("foo")
- .setTaskModule("x2.y2")
+ .addOutEdgeId("e1")
.build()
- )
- .addInEdgeId("e1")
- .build()
- )
+ )
+ .addVertex(
+ VertexPlan.newBuilder()
+ .setName("vertex2")
+ .setType(PlanVertexType.NORMAL)
+ .setProcessorDescriptor(
+ TezEntityDescriptorProto.newBuilder().setClassName("x2.y2"))
+ .addTaskLocationHint(
+ PlanTaskLocationHint.newBuilder()
+ .addHost("host2")
+ .addRack("rack2")
+ .build()
+ )
+ .setTaskConfig(
+ PlanTaskConfiguration.newBuilder()
+ .setNumTasks(2)
+ .setVirtualCores(4)
+ .setMemoryMb(1024)
+ .setJavaOpts("foo")
+ .setTaskModule("x2.y2")
+ .build()
+ )
+ .addInEdgeId("e1")
+ .build()
+ )
.addEdge(
- EdgePlan.newBuilder()
- .setEdgeManager(TezEntityDescriptorProto.newBuilder()
- .setClassName(CustomizedEdgeManager.class.getName())
- .setUserPayload(ByteString.copyFromUtf8(exLocation.name()))
- )
- .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("v1_v2"))
- .setInputVertexName("vertex1")
- .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o1"))
- .setOutputVertexName("vertex2")
- .setDataMovementType(PlanEdgeDataMovementType.CUSTOM)
- .setId("e1")
- .setDataSourceType(PlanEdgeDataSourceType.PERSISTED)
- .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL)
- .build()
- )
+ EdgePlan.newBuilder()
+ .setEdgeManager(TezEntityDescriptorProto.newBuilder()
+ .setClassName(CustomizedEdgeManager.class.getName())
+ .setTezUserPayload(DAGProtos.TezUserPayloadProto.newBuilder()
+ .setUserPayload(ByteString.copyFromUtf8(exLocation.name())))
+ )
+ .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("v1_v2"))
+ .setInputVertexName("vertex1")
+ .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o1"))
+ .setOutputVertexName("vertex2")
+ .setDataMovementType(PlanEdgeDataMovementType.CUSTOM)
+ .setId("e1")
+ .setDataSourceType(PlanEdgeDataSourceType.PERSISTED)
+ .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL)
+ .build()
+ )
.build();
return dag;
}
@@ -1135,14 +1137,15 @@ public class TestDAGImpl {
TezEntityDescriptorProto
.newBuilder()
.setClassName(CountingOutputCommitter.class.getName())
+ .setTezUserPayload(DAGProtos.TezUserPayloadProto.newBuilder()
.setUserPayload(
ByteString
.copyFrom(new CountingOutputCommitter.CountingOutputCommitterConfig(
- true, false, false).toUserPayload())).build())
+ true, false, false).toUserPayload())).build()))
.setName("output3")
.setIODescriptor(
TezEntityDescriptorProto.newBuilder().setClassName("output.class")
- )
+ )
.build());
badVertex.setAdditionalOutputs(outputs);
http://git-wip-us.apache.org/repos/asf/tez/blob/4e69bed5/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
index 72dcd92..55ee05f 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
@@ -219,7 +219,7 @@ public class TestVertexImpl {
@Override
public void initialize() throws IOException {
- if (getContext().getUserPayload().hasPayload()) {
+ if (getContext().getUserPayload() != null && getContext().getUserPayload().hasPayload()) {
CountingOutputCommitterConfig conf =
new CountingOutputCommitterConfig(getContext().getUserPayload());
this.throwError = conf.throwError;
@@ -419,8 +419,9 @@ public class TestVertexImpl {
)
.addOutEdgeId("e1")
.setVertexManagerPlugin(TezEntityDescriptorProto.newBuilder()
- .setClassName(VertexManagerWithException.class.getName())
- .setUserPayload(ByteString.copyFrom(exLocation.name().getBytes())))
+ .setClassName(VertexManagerWithException.class.getName())
+ .setTezUserPayload(DAGProtos.TezUserPayloadProto.newBuilder()
+ .setUserPayload(ByteString.copyFrom(exLocation.name().getBytes()))))
.build()
)
.addVertex(
@@ -438,8 +439,9 @@ public class TestVertexImpl {
)
.addInEdgeId("e1")
.setVertexManagerPlugin(TezEntityDescriptorProto.newBuilder()
- .setClassName(VertexManagerWithException.class.getName())
- .setUserPayload(ByteString.copyFrom(exLocation.name().getBytes())))
+ .setClassName(VertexManagerWithException.class.getName())
+ .setTezUserPayload(DAGProtos.TezUserPayloadProto.newBuilder()
+ .setUserPayload(ByteString.copyFrom(exLocation.name().getBytes()))))
.build()
)
.addEdge(
@@ -706,24 +708,24 @@ public class TestVertexImpl {
.setControllerDescriptor(
TezEntityDescriptorProto.newBuilder().setClassName(
initializerClassName))
- .setName("input1")
- .setIODescriptor(
- TezEntityDescriptorProto.newBuilder()
- .setClassName("InputClazz")
- .build()
- ).build()
+ .setName("input1")
+ .setIODescriptor(
+ TezEntityDescriptorProto.newBuilder()
+ .setClassName("InputClazz")
+ .build()
+ ).build()
)
.setTaskConfig(
PlanTaskConfiguration.newBuilder()
- .setNumTasks(-1)
- .setVirtualCores(4)
- .setMemoryMb(1024)
- .setJavaOpts("")
- .setTaskModule("x1.y1")
- .build()
+ .setNumTasks(-1)
+ .setVirtualCores(4)
+ .setMemoryMb(1024)
+ .setJavaOpts("")
+ .setTaskModule("x1.y1")
+ .build()
)
.addOutEdgeId("e1")
- .build()
+ .build()
)
.addVertex(
VertexPlan.newBuilder()
@@ -737,11 +739,11 @@ public class TestVertexImpl {
.setName("input2")
.setIODescriptor(
TezEntityDescriptorProto.newBuilder()
- .setClassName("InputClazz")
- .build()
+ .setClassName("InputClazz")
+ .build()
)
.build()
- )
+ )
.setTaskConfig(
PlanTaskConfiguration.newBuilder()
.setNumTasks(-1)
@@ -752,7 +754,7 @@ public class TestVertexImpl {
.build()
)
.addInEdgeId("e1")
- .build()
+ .build()
)
.addVertex(
VertexPlan.newBuilder()
@@ -766,8 +768,8 @@ public class TestVertexImpl {
.setName("input3")
.setIODescriptor(
TezEntityDescriptorProto.newBuilder()
- .setClassName("InputClazz")
- .build()
+ .setClassName("InputClazz")
+ .build()
)
.build()
)
@@ -782,40 +784,42 @@ public class TestVertexImpl {
)
.setVertexManagerPlugin(TezEntityDescriptorProto.newBuilder()
.setClassName(RootInputSpecUpdaterVertexManager.class.getName())
- .setUserPayload(ByteString.copyFrom(new byte[] {0})))
+ .setTezUserPayload(DAGProtos.TezUserPayloadProto.newBuilder()
+ .setUserPayload(ByteString.copyFrom(new byte[]{0}))))
.build()
)
.addVertex(
- VertexPlan.newBuilder()
- .setName("vertex4")
- .setType(PlanVertexType.NORMAL)
- .addInputs(
- RootInputLeafOutputProto.newBuilder()
- .setControllerDescriptor(
- TezEntityDescriptorProto.newBuilder().setClassName(
- initializerClassName))
- .setName("input4")
- .setIODescriptor(
- TezEntityDescriptorProto.newBuilder()
- .setClassName("InputClazz")
- .build()
+ VertexPlan.newBuilder()
+ .setName("vertex4")
+ .setType(PlanVertexType.NORMAL)
+ .addInputs(
+ RootInputLeafOutputProto.newBuilder()
+ .setControllerDescriptor(
+ TezEntityDescriptorProto.newBuilder().setClassName(
+ initializerClassName))
+ .setName("input4")
+ .setIODescriptor(
+ TezEntityDescriptorProto.newBuilder()
+ .setClassName("InputClazz")
+ .build()
+ )
+ .build()
)
- .build()
- )
- .setTaskConfig(
- PlanTaskConfiguration.newBuilder()
- .setNumTasks(-1)
- .setVirtualCores(4)
- .setMemoryMb(1024)
- .setJavaOpts("")
- .setTaskModule("x3.y3")
+ .setTaskConfig(
+ PlanTaskConfiguration.newBuilder()
+ .setNumTasks(-1)
+ .setVirtualCores(4)
+ .setMemoryMb(1024)
+ .setJavaOpts("")
+ .setTaskModule("x3.y3")
+ .build()
+ )
+ .setVertexManagerPlugin(TezEntityDescriptorProto.newBuilder()
+ .setClassName(RootInputSpecUpdaterVertexManager.class.getName())
+ .setTezUserPayload(DAGProtos.TezUserPayloadProto.newBuilder()
+ .setUserPayload(ByteString.copyFrom(new byte[]{1}))))
.build()
)
- .setVertexManagerPlugin(TezEntityDescriptorProto.newBuilder()
- .setClassName(RootInputSpecUpdaterVertexManager.class.getName())
- .setUserPayload(ByteString.copyFrom(new byte[] {1})))
- .build()
- )
.addEdge(
EdgePlan.newBuilder()
.setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("v1_v2"))
@@ -827,7 +831,7 @@ public class TestVertexImpl {
.setDataSourceType(PlanEdgeDataSourceType.PERSISTED)
.setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL)
.build()
- )
+ )
.build();
return dag;
}
@@ -1404,71 +1408,71 @@ public class TestVertexImpl {
.build()
)
.addVertex(
- VertexPlan.newBuilder()
- .setName("vertex2")
- .setType(PlanVertexType.NORMAL)
- .addTaskLocationHint(
- PlanTaskLocationHint.newBuilder()
- .addHost("host2")
- .addRack("rack2")
- .build()
- )
- .setTaskConfig(
- PlanTaskConfiguration.newBuilder()
- .setNumTasks(2)
- .setVirtualCores(4)
- .setMemoryMb(1024)
- .setJavaOpts("")
- .setTaskModule("x2.y2")
+ VertexPlan.newBuilder()
+ .setName("vertex2")
+ .setType(PlanVertexType.NORMAL)
+ .addTaskLocationHint(
+ PlanTaskLocationHint.newBuilder()
+ .addHost("host2")
+ .addRack("rack2")
+ .build()
+ )
+ .setTaskConfig(
+ PlanTaskConfiguration.newBuilder()
+ .setNumTasks(2)
+ .setVirtualCores(4)
+ .setMemoryMb(1024)
+ .setJavaOpts("")
+ .setTaskModule("x2.y2")
+ .build()
+ )
+ .addOutEdgeId("e2")
.build()
- )
- .addOutEdgeId("e2")
- .build()
)
.addVertex(
- VertexPlan.newBuilder()
- .setName("vertex3")
- .setType(PlanVertexType.NORMAL)
- .setProcessorDescriptor(TezEntityDescriptorProto.newBuilder().setClassName("x3.y3"))
- .addTaskLocationHint(
- PlanTaskLocationHint.newBuilder()
- .addHost("host3")
- .addRack("rack3")
- .build()
- )
- .setTaskConfig(
- PlanTaskConfiguration.newBuilder()
- .setNumTasks(2)
- .setVirtualCores(4)
- .setMemoryMb(1024)
- .setJavaOpts("foo")
- .setTaskModule("x3.y3")
+ VertexPlan.newBuilder()
+ .setName("vertex3")
+ .setType(PlanVertexType.NORMAL)
+ .setProcessorDescriptor(TezEntityDescriptorProto.newBuilder().setClassName("x3.y3"))
+ .addTaskLocationHint(
+ PlanTaskLocationHint.newBuilder()
+ .addHost("host3")
+ .addRack("rack3")
+ .build()
+ )
+ .setTaskConfig(
+ PlanTaskConfiguration.newBuilder()
+ .setNumTasks(2)
+ .setVirtualCores(4)
+ .setMemoryMb(1024)
+ .setJavaOpts("foo")
+ .setTaskModule("x3.y3")
+ .build()
+ )
+ .addInEdgeId("e1")
+ .addInEdgeId("e2")
+ .addOutEdgeId("e3")
+ .addOutEdgeId("e4")
.build()
- )
- .addInEdgeId("e1")
- .addInEdgeId("e2")
- .addOutEdgeId("e3")
- .addOutEdgeId("e4")
- .build()
)
.addVertex(
VertexPlan.newBuilder()
.setName("vertex4")
.setType(PlanVertexType.NORMAL)
.addTaskLocationHint(
- PlanTaskLocationHint.newBuilder()
- .addHost("host4")
- .addRack("rack4")
- .build()
+ PlanTaskLocationHint.newBuilder()
+ .addHost("host4")
+ .addRack("rack4")
+ .build()
)
.setTaskConfig(
- PlanTaskConfiguration.newBuilder()
- .setNumTasks(2)
- .setVirtualCores(4)
- .setMemoryMb(1024)
- .setJavaOpts("")
- .setTaskModule("x4.y4")
- .build()
+ PlanTaskConfiguration.newBuilder()
+ .setNumTasks(2)
+ .setVirtualCores(4)
+ .setMemoryMb(1024)
+ .setJavaOpts("")
+ .setTaskModule("x4.y4")
+ .build()
)
.addInEdgeId("e3")
.addOutEdgeId("e5")
@@ -1479,19 +1483,19 @@ public class TestVertexImpl {
.setName("vertex5")
.setType(PlanVertexType.NORMAL)
.addTaskLocationHint(
- PlanTaskLocationHint.newBuilder()
- .addHost("host5")
- .addRack("rack5")
- .build()
+ PlanTaskLocationHint.newBuilder()
+ .addHost("host5")
+ .addRack("rack5")
+ .build()
)
.setTaskConfig(
- PlanTaskConfiguration.newBuilder()
- .setNumTasks(2)
- .setVirtualCores(4)
- .setMemoryMb(1024)
- .setJavaOpts("")
- .setTaskModule("x5.y5")
- .build()
+ PlanTaskConfiguration.newBuilder()
+ .setNumTasks(2)
+ .setVirtualCores(4)
+ .setMemoryMb(1024)
+ .setJavaOpts("")
+ .setTaskModule("x5.y5")
+ .build()
)
.addInEdgeId("e4")
.addOutEdgeId("e6")
@@ -1575,9 +1579,10 @@ public class TestVertexImpl {
.setDataMovementType(PlanEdgeDataMovementType.CUSTOM)
.setEdgeManager(
TezEntityDescriptorProto.newBuilder()
- .setClassName(EdgeManagerForTest.class.getName())
- .setUserPayload(ByteString.copyFrom(edgePayload))
- .build())
+ .setClassName(EdgeManagerForTest.class.getName())
+ .setTezUserPayload(DAGProtos.TezUserPayloadProto.newBuilder()
+ .setUserPayload((ByteString.copyFrom(edgePayload))))
+ .build())
.setId("e4")
.setDataSourceType(PlanEdgeDataSourceType.PERSISTED)
.setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL)
@@ -2833,9 +2838,11 @@ public class TestVertexImpl {
outputs.add(RootInputLeafOutputProto.newBuilder()
.setControllerDescriptor(
TezEntityDescriptorProto.newBuilder().setClassName(
- CountingOutputCommitter.class.getName()).setUserPayload(ByteString.copyFrom(
- new CountingOutputCommitter.CountingOutputCommitterConfig()
- .toUserPayload())).build())
+ CountingOutputCommitter.class.getName())
+ .setTezUserPayload(DAGProtos.TezUserPayloadProto.newBuilder()
+ .setUserPayload(ByteString.copyFrom(
+ new CountingOutputCommitter.CountingOutputCommitterConfig()
+ .toUserPayload())).build()))
.setName("output_v2")
.setIODescriptor(
TezEntityDescriptorProto.newBuilder().setClassName("output.class"))
@@ -2946,9 +2953,11 @@ public class TestVertexImpl {
outputs.add(RootInputLeafOutputProto.newBuilder()
.setControllerDescriptor(
TezEntityDescriptorProto.newBuilder().setClassName(
- CountingOutputCommitter.class.getName()).setUserPayload(ByteString.copyFrom(
+ CountingOutputCommitter.class.getName())
+ .setTezUserPayload(DAGProtos.TezUserPayloadProto.newBuilder()
+ .setUserPayload(ByteString.copyFrom(
new CountingOutputCommitter.CountingOutputCommitterConfig(
- true, true, false).toUserPayload())).build())
+ true, true, false).toUserPayload())).build()))
.setName("output_v2")
.setIODescriptor(
TezEntityDescriptorProto.newBuilder().setClassName("output.class"))
@@ -2990,9 +2999,11 @@ public class TestVertexImpl {
outputs.add(RootInputLeafOutputProto.newBuilder()
.setControllerDescriptor(
TezEntityDescriptorProto.newBuilder().setClassName(
- CountingOutputCommitter.class.getName()).setUserPayload(ByteString.copyFrom(
+ CountingOutputCommitter.class.getName())
+ .setTezUserPayload(DAGProtos.TezUserPayloadProto.newBuilder()
+ .setUserPayload(ByteString.copyFrom(
new CountingOutputCommitter.CountingOutputCommitterConfig(
- true, true, true).toUserPayload())).build())
+ true, true, true).toUserPayload())).build()))
.setName("output_v2")
.setIODescriptor(
TezEntityDescriptorProto.newBuilder().setClassName("output.class"))
@@ -4162,7 +4173,8 @@ public class TestVertexImpl {
.setEdgeManager(
TezEntityDescriptorProto.newBuilder()
.setClassName(EdgeManagerForTest.class.getName())
- .setUserPayload(ByteString.copyFrom(edgePayload))
+ .setTezUserPayload(DAGProtos.TezUserPayloadProto.newBuilder()
+ .setUserPayload(ByteString.copyFrom(edgePayload)))
.build())
.setOutputVertexName("M5")
.setDataMovementType(PlanEdgeDataMovementType.CUSTOM)
http://git-wip-us.apache.org/repos/asf/tez/blob/4e69bed5/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java b/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java
index 903b4fe..ad508b6 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java
@@ -325,8 +325,7 @@ public class TestHistoryEventsProtoConversion {
deserializedEvent.getSourceEdgeManagers().size());
Assert.assertEquals(event.getSourceEdgeManagers().get("foo").getClassName(),
deserializedEvent.getSourceEdgeManagers().get("foo").getClassName());
- Assert.assertNotNull(deserializedEvent.getSourceEdgeManagers().get("foo").getUserPayload());
- Assert.assertNull(deserializedEvent.getSourceEdgeManagers().get("foo").getUserPayload().getPayload());
+ Assert.assertNull(deserializedEvent.getSourceEdgeManagers().get("foo").getUserPayload());
Assert.assertEquals(event.getSourceEdgeManagers().get("foo1").getClassName(),
deserializedEvent.getSourceEdgeManagers().get("foo1").getClassName());
Assert.assertEquals(event.getSourceEdgeManagers().get("foo1").getUserPayload().getVersion(),
http://git-wip-us.apache.org/repos/asf/tez/blob/4e69bed5/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezInputContextImpl.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezInputContextImpl.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezInputContextImpl.java
index 420c477..62311e9 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezInputContextImpl.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezInputContextImpl.java
@@ -76,7 +76,7 @@ public class TezInputContextImpl extends TezTaskContextImpl
checkNotNull(sourceVertexName, "sourceVertexName is null");
checkNotNull(inputs, "input map is null");
checkNotNull(inputReadyTracker, "inputReadyTracker is null");
- this.userPayload = userPayload == null ? UserPayload.create(null) : userPayload;
+ this.userPayload = userPayload;
this.inputIndex = inputIndex;
this.sourceVertexName = sourceVertexName;
this.sourceInfo = new EventMetaData(
http://git-wip-us.apache.org/repos/asf/tez/blob/4e69bed5/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezMergedInputContextImpl.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezMergedInputContextImpl.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezMergedInputContextImpl.java
index 693bde8..e71a0bc 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezMergedInputContextImpl.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezMergedInputContextImpl.java
@@ -46,7 +46,7 @@ public class TezMergedInputContextImpl implements MergedInputContext {
checkNotNull(inputReadyTracker, "inputReadyTracker is null");
this.groupInputName = groupInputName;
this.groupInputsMap = groupInputsMap;
- this.userPayload = userPayload == null ? UserPayload.create(null) : userPayload;
+ this.userPayload = userPayload;
this.inputReadyTracker = inputReadyTracker;
this.workDirs = workDirs;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/4e69bed5/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezOutputContextImpl.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezOutputContextImpl.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezOutputContextImpl.java
index 94df6aa..7b53075 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezOutputContextImpl.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezOutputContextImpl.java
@@ -69,7 +69,7 @@ public class TezOutputContextImpl extends TezTaskContextImpl
auxServiceEnv, memDist, outputDescriptor, objectRegistry);
checkNotNull(outputIndex, "outputIndex is null");
checkNotNull(destinationVertexName, "destinationVertexName is null");
- this.userPayload = userPayload == null ? UserPayload.create(null) : userPayload;
+ this.userPayload = userPayload;
this.outputIndex = outputIndex;
this.destinationVertexName = destinationVertexName;
this.sourceInfo = new EventMetaData(EventProducerConsumerType.OUTPUT,
http://git-wip-us.apache.org/repos/asf/tez/blob/4e69bed5/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezProcessorContextImpl.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezProcessorContextImpl.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezProcessorContextImpl.java
index 330c4c9..9b4dc6a 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezProcessorContextImpl.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezProcessorContextImpl.java
@@ -62,7 +62,7 @@ public class TezProcessorContextImpl extends TezTaskContextImpl implements Proce
counters, runtimeTask, tezUmbilical, serviceConsumerMetadata,
auxServiceEnv, memDist, processorDescriptor, objectRegistry);
checkNotNull(inputReadyTracker, "inputReadyTracker is null");
- this.userPayload = userPayload == null ? UserPayload.create(null) : userPayload;
+ this.userPayload = userPayload;
this.sourceInfo = new EventMetaData(EventProducerConsumerType.PROCESSOR,
taskVertexName, "", taskAttemptID);
this.inputReadyTracker = inputReadyTracker;
http://git-wip-us.apache.org/repos/asf/tez/blob/4e69bed5/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
index e8a137c..42b31f1 100644
--- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
@@ -72,7 +72,8 @@ public class BroadcastAndOneToOneExample extends Configured implements Tool {
.next();
KeyValueWriter kvWriter = (KeyValueWriter) output.getWriter();
kvWriter.write(word, new IntWritable(getContext().getTaskIndex()));
- ByteBuffer userPayload = getContext().getUserPayload().getPayload();
+ ByteBuffer userPayload =
+ getContext().getUserPayload() == null ? null : getContext().getUserPayload().getPayload();
if (userPayload != null) {
boolean doLocalityCheck = getContext().getUserPayload().getPayload().get(0) > 0 ? true : false;
if (doLocalityCheck) {
[29/50] [abbrv] tez git commit: TEZ-1749. Increase test timeout for
TestLocalMode.testMultipleClientsWithSession (Rajesh Balamohan)
Posted by je...@apache.org.
TEZ-1749. Increase test timeout for TestLocalMode.testMultipleClientsWithSession (Rajesh Balamohan)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/b810c7db
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/b810c7db
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/b810c7db
Branch: refs/heads/TEZ-8
Commit: b810c7db18dbe7d43d1df517a556af5e56c507fd
Parents: 873b247
Author: Rajesh Balamohan <rb...@apache.org>
Authored: Thu Nov 6 20:36:57 2014 -0800
Committer: Rajesh Balamohan <rb...@apache.org>
Committed: Thu Nov 6 20:36:57 2014 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
tez-dag/src/test/java/org/apache/tez/client/TestLocalMode.java | 2 +-
2 files changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/b810c7db/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 359f43c..bac35ff 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,7 @@ ALL CHANGES:
TEZ-1733. TezMerger should sort FileChunks on size when merging
TEZ-1738. Tez tfile parser for log parsing
TEZ-1627. Remove OUTPUT_CONSUMABLE and related Event in TaskAttemptImpl
+ TEZ-1749. Increase test timeout for TestLocalMode.testMultipleClientsWithSession
Release 0.5.3: Unreleased
http://git-wip-us.apache.org/repos/asf/tez/blob/b810c7db/tez-dag/src/test/java/org/apache/tez/client/TestLocalMode.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/client/TestLocalMode.java b/tez-dag/src/test/java/org/apache/tez/client/TestLocalMode.java
index cd8a3de..7d50c06 100644
--- a/tez-dag/src/test/java/org/apache/tez/client/TestLocalMode.java
+++ b/tez-dag/src/test/java/org/apache/tez/client/TestLocalMode.java
@@ -44,7 +44,7 @@ import org.junit.Test;
public class TestLocalMode {
- @Test(timeout = 10000)
+ @Test(timeout = 30000)
public void testMultipleClientsWithSession() throws TezException, InterruptedException,
IOException {
TezConfiguration tezConf1 = new TezConfiguration();
[31/50] [abbrv] tez git commit: TEZ-1742. Addendum to fix CHANGES.txt
Posted by je...@apache.org.
TEZ-1742. Addendum to fix CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/8492f107
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/8492f107
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/8492f107
Branch: refs/heads/TEZ-8
Commit: 8492f10757f9134016f8149f2cc54061d8ddbb0a
Parents: 5df9004
Author: Bikas Saha <bi...@apache.org>
Authored: Fri Nov 7 08:52:53 2014 -0800
Committer: Bikas Saha <bi...@apache.org>
Committed: Fri Nov 7 08:52:53 2014 -0800
----------------------------------------------------------------------
CHANGES.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/8492f107/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index f4742f9..5749ea2 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -10,11 +10,11 @@ ALL CHANGES:
TEZ-1738. Tez tfile parser for log parsing
TEZ-1627. Remove OUTPUT_CONSUMABLE and related Event in TaskAttemptImpl
TEZ-1749. Increase test timeout for TestLocalMode.testMultipleClientsWithSession
- TEZ-1742. Improve response time of internal preemption
Release 0.5.3: Unreleased
ALL CHANGES:
+ TEZ-1742. Improve response time of internal preemption
TEZ-1745. TestATSHistoryLoggingService::testATSHistoryLoggingServiceShutdown can be flaky.
TEZ-1747. Increase test timeout for TestSecureShuffle.
TEZ-1746. Flaky test in TestVertexImpl and TestExceptionPropagation.
[13/50] [abbrv] tez git commit: TEZ-1698. addendum - fix license
template issue. (Rajesh Balamohan)
Posted by je...@apache.org.
TEZ-1698. addendum - fix license template issue. (Rajesh Balamohan)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/81322efc
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/81322efc
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/81322efc
Branch: refs/heads/TEZ-8
Commit: 81322efc2b8b843008cb3bebe8e5db78af20eb58
Parents: 8f8a81f
Author: Rajesh Balamohan <rb...@apache.org>
Authored: Fri Oct 31 12:17:31 2014 +0530
Committer: Rajesh Balamohan <rb...@apache.org>
Committed: Fri Oct 31 12:17:31 2014 +0530
----------------------------------------------------------------------
.../tez/util/TezMxBeanResourceCalculator.java | 22 ++++++------
.../util/TestTezMxBeanResourceCalculator.java | 36 ++++++++++----------
2 files changed, 29 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/81322efc/tez-plugins/tez-mbeans-resource-calculator/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-mbeans-resource-calculator/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java b/tez-plugins/tez-mbeans-resource-calculator/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java
index e8aeb0b..330a2fd 100644
--- a/tez-plugins/tez-mbeans-resource-calculator/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java
+++ b/tez-plugins/tez-mbeans-resource-calculator/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java
@@ -1,11 +1,3 @@
-package org.apache.tez.util;
-
-import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
-
-import java.lang.management.ManagementFactory;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -14,9 +6,9 @@ import java.util.concurrent.atomic.AtomicLong;
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -24,6 +16,14 @@ import java.util.concurrent.atomic.AtomicLong;
* limitations under the License.
*/
+package org.apache.tez.util;
+
+import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
+
+import java.lang.management.ManagementFactory;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
/**
* Uses sun's MBeans to return process information.
*/
http://git-wip-us.apache.org/repos/asf/tez/blob/81322efc/tez-plugins/tez-mbeans-resource-calculator/src/test/java/org/apache/tez/util/TestTezMxBeanResourceCalculator.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-mbeans-resource-calculator/src/test/java/org/apache/tez/util/TestTezMxBeanResourceCalculator.java b/tez-plugins/tez-mbeans-resource-calculator/src/test/java/org/apache/tez/util/TestTezMxBeanResourceCalculator.java
index ade2118..fdc0cb7 100644
--- a/tez-plugins/tez-mbeans-resource-calculator/src/test/java/org/apache/tez/util/TestTezMxBeanResourceCalculator.java
+++ b/tez-plugins/tez-mbeans-resource-calculator/src/test/java/org/apache/tez/util/TestTezMxBeanResourceCalculator.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.tez.util;
import org.apache.hadoop.conf.Configuration;
@@ -21,24 +39,6 @@ import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
public class TestTezMxBeanResourceCalculator {
private ResourceCalculatorProcessTree resourceCalculator;
[08/50] [abbrv] tez git commit: TEZ-1722. DAG should be related to
Application Id in ATS data. (hitesh)
Posted by je...@apache.org.
TEZ-1722. DAG should be related to Application Id in ATS data. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/c570a0e1
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/c570a0e1
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/c570a0e1
Branch: refs/heads/TEZ-8
Commit: c570a0e12921fff5787877ceb5612b08115d77a4
Parents: ede0e64
Author: Hitesh Shah <hi...@apache.org>
Authored: Thu Oct 30 16:23:27 2014 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Thu Oct 30 16:23:27 2014 -0700
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../dag/history/logging/impl/HistoryEventJsonConversion.java | 8 +++++++-
.../history/logging/ats/HistoryEventTimelineConversion.java | 2 ++
.../logging/ats/TestHistoryEventTimelineConversion.java | 5 ++++-
4 files changed, 14 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/c570a0e1/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 5d6ebf5..68d4838 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -90,6 +90,7 @@ ALL CHANGES:
TEZ-1700. Replace containerId from TaskLocationHint with [TaskIndex+Vertex]
based affinity
TEZ-1716. Additional ATS data for UI.
+ TEZ-1722. DAG should be related to Application Id in ATS data.
Release 0.5.1: 2014-10-02
http://git-wip-us.apache.org/repos/asf/tez/blob/c570a0e1/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/HistoryEventJsonConversion.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/HistoryEventJsonConversion.java b/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/HistoryEventJsonConversion.java
index 0b6f9d2..8560359 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/HistoryEventJsonConversion.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/HistoryEventJsonConversion.java
@@ -409,8 +409,13 @@ public class HistoryEventJsonConversion {
JSONArray relatedEntities = new JSONArray();
JSONObject tezAppEntity = new JSONObject();
tezAppEntity.put(ATSConstants.ENTITY,
- "tez_" + event.getApplicationAttemptId().toString());
+ "tez_" + event.getApplicationAttemptId().getApplicationId().toString());
tezAppEntity.put(ATSConstants.ENTITY_TYPE,
+ EntityTypes.TEZ_APPLICATION.name());
+ JSONObject tezAppAttemptEntity = new JSONObject();
+ tezAppAttemptEntity.put(ATSConstants.ENTITY,
+ "tez_" + event.getApplicationAttemptId().toString());
+ tezAppAttemptEntity.put(ATSConstants.ENTITY_TYPE,
EntityTypes.TEZ_APPLICATION_ATTEMPT.name());
JSONObject appEntity = new JSONObject();
appEntity.put(ATSConstants.ENTITY,
@@ -429,6 +434,7 @@ public class HistoryEventJsonConversion {
ATSConstants.USER);
relatedEntities.put(tezAppEntity);
+ relatedEntities.put(tezAppAttemptEntity);
relatedEntities.put(appEntity);
relatedEntities.put(appAttemptEntity);
relatedEntities.put(userEntity);
http://git-wip-us.apache.org/repos/asf/tez/blob/c570a0e1/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
index a492408..865915e 100644
--- a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
+++ b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java
@@ -318,6 +318,8 @@ public class HistoryEventTimelineConversion {
atsEntity.setEntityId(event.getDagID().toString());
atsEntity.setEntityType(EntityTypes.TEZ_DAG_ID.name());
+ atsEntity.addRelatedEntity(EntityTypes.TEZ_APPLICATION.name(),
+ "tez_" + event.getApplicationAttemptId().getApplicationId().toString());
atsEntity.addRelatedEntity(EntityTypes.TEZ_APPLICATION_ATTEMPT.name(),
"tez_" + event.getApplicationAttemptId().toString());
atsEntity.addRelatedEntity(ATSConstants.APPLICATION_ID,
http://git-wip-us.apache.org/repos/asf/tez/blob/c570a0e1/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
index ba71d46..2fed18b 100644
--- a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
+++ b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
@@ -267,7 +267,10 @@ public class TestHistoryEventTimelineConversion {
Assert.assertEquals(EntityTypes.TEZ_DAG_ID.name(), timelineEntity.getEntityType());
Assert.assertEquals(tezDAGID.toString(), timelineEntity.getEntityId());
- Assert.assertEquals(4, timelineEntity.getRelatedEntities().size());
+ Assert.assertEquals(5, timelineEntity.getRelatedEntities().size());
+ Assert.assertTrue(
+ timelineEntity.getRelatedEntities().get(EntityTypes.TEZ_APPLICATION.name()).contains(
+ "tez_" + applicationId.toString()));
Assert.assertTrue(
timelineEntity.getRelatedEntities().get(EntityTypes.TEZ_APPLICATION_ATTEMPT.name()).contains(
"tez_" + applicationAttemptId.toString()));
[30/50] [abbrv] tez git commit: TEZ-1742. Improve response time of
internal preemption (bikas)
Posted by je...@apache.org.
TEZ-1742. Improve response time of internal preemption (bikas)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/5df90046
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/5df90046
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/5df90046
Branch: refs/heads/TEZ-8
Commit: 5df90046c494352b55dbffe2fbd6810516d486e7
Parents: b810c7d
Author: Bikas Saha <bi...@apache.org>
Authored: Fri Nov 7 08:39:34 2014 -0800
Committer: Bikas Saha <bi...@apache.org>
Committed: Fri Nov 7 08:39:34 2014 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/tez/dag/api/TezConfiguration.java | 22 +++
.../dag/app/rm/YarnTaskSchedulerService.java | 193 ++++++++++++++-----
.../tez/dag/app/rm/TestTaskScheduler.java | 67 +++++--
4 files changed, 215 insertions(+), 68 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/5df90046/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index bac35ff..f4742f9 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -10,6 +10,7 @@ ALL CHANGES:
TEZ-1738. Tez tfile parser for log parsing
TEZ-1627. Remove OUTPUT_CONSUMABLE and related Event in TaskAttemptImpl
TEZ-1749. Increase test timeout for TestLocalMode.testMultipleClientsWithSession
+ TEZ-1742. Improve response time of internal preemption
Release 0.5.3: Unreleased
http://git-wip-us.apache.org/repos/asf/tez/blob/5df90046/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 7cc0aa5..d9003b3 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -570,6 +570,28 @@ public class TezConfiguration extends Configuration {
public static final String TEZ_AM_SESSION_MIN_HELD_CONTAINERS =
TEZ_AM_PREFIX + "session.min.held-containers";
public static final int TEZ_AM_SESSION_MIN_HELD_CONTAINERS_DEFAULT = 0;
+
+ /**
+ * Int value. Specifies the percentage of tasks eligible to be preempted that
+ * will actually be preempted in a given round of Tez internal preemption.
+ * This slows down preemption and gives more time for free resources to be
+ * allocated by the cluster (if any) and gives more time for preemptable tasks
+ * to finish. Valid values are 0-100. Higher values will preempt quickly at
+ * the cost of losing work. Setting to 0 turns off preemption. Expert level
+ * setting.
+ */
+ public static final String TEZ_AM_PREEMPTION_PERCENTAGE =
+ TEZ_AM_PREFIX + "preemption.percentage";
+ public static final int TEZ_AM_PREEMPTION_PERCENTAGE_DEFAULT = 10;
+
+ /**
+ * Int value. The number of RM heartbeats to wait after preempting running tasks before preempting
+ * more running tasks. After preempting a task, we need to wait at least 1 heartbeat so that the
+ * RM can act on the released resources and assign new ones to us. Expert level setting.
+ */
+ public static final String TEZ_AM_PREEMPTION_HEARTBEATS_BETWEEN_PREEMPTIONS =
+ TEZ_AM_PREFIX + "preemption.heartbeats-between-preemptions";
+ public static final int TEZ_AM_PREEMPTION_HEARTBEATS_BETWEEN_PREEMPTIONS_DEFAULT = 3;
/**
* String value to a file path.
http://git-wip-us.apache.org/repos/asf/tez/blob/5df90046/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java
index 75d62f1..5941a45 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java
@@ -123,6 +123,9 @@ public class YarnTaskSchedulerService extends TaskSchedulerService
Resource totalResources = Resource.newInstance(0, 0);
Resource allocatedResources = Resource.newInstance(0, 0);
+ long numHeartbeats = 0;
+ long heartbeatAtLastPreemption = 0;
+ int numHeartbeatsBetweenPreemptions = 0;
final String appHostName;
final int appHostPort;
@@ -141,6 +144,8 @@ public class YarnTaskSchedulerService extends TaskSchedulerService
long idleContainerTimeoutMin;
long idleContainerTimeoutMax = 0;
int sessionNumMinHeldContainers = 0;
+ int preemptionPercentage = 0;
+
Set<ContainerId> sessionMinHeldContainers = Sets.newHashSet();
RandomDataGenerator random = new RandomDataGenerator();
@@ -328,6 +333,17 @@ public class YarnTaskSchedulerService extends TaskSchedulerService
TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS_DEFAULT);
Preconditions.checkArgument(sessionNumMinHeldContainers >= 0,
"Session minimum held containers should be >=0");
+
+ preemptionPercentage = conf.getInt(TezConfiguration.TEZ_AM_PREEMPTION_PERCENTAGE,
+ TezConfiguration.TEZ_AM_PREEMPTION_PERCENTAGE_DEFAULT);
+ Preconditions.checkArgument(preemptionPercentage >= 0 && preemptionPercentage <= 100,
+ "Preemption percentage should be between 0-100");
+
+ numHeartbeatsBetweenPreemptions = conf.getInt(
+ TezConfiguration.TEZ_AM_PREEMPTION_HEARTBEATS_BETWEEN_PREEMPTIONS,
+ TezConfiguration.TEZ_AM_PREEMPTION_HEARTBEATS_BETWEEN_PREEMPTIONS_DEFAULT);
+ Preconditions.checkArgument(numHeartbeatsBetweenPreemptions >= 1,
+ "Heartbeats between preemptions should be >=1");
delayedContainerManager = new DelayedContainerManager();
LOG.info("TaskScheduler initialized with configuration: " +
@@ -336,6 +352,8 @@ public class YarnTaskSchedulerService extends TaskSchedulerService
", reuseRackLocal: " + reuseRackLocal +
", reuseNonLocal: " + reuseNonLocal +
", localitySchedulingDelay: " + localitySchedulingDelay +
+ ", preemptionPercentage: " + preemptionPercentage +
+ ", numHeartbeatsBetweenPreemptions" + numHeartbeatsBetweenPreemptions +
", idleContainerMinTimeout=" + idleContainerTimeoutMin +
", idleContainerMaxTimeout=" + idleContainerTimeoutMax +
", sessionMinHeldContainers=" + sessionNumMinHeldContainers);
@@ -847,6 +865,7 @@ public class YarnTaskSchedulerService extends TaskSchedulerService
" taskAllocations: " + taskAllocations.size());
}
+ numHeartbeats++;
preemptIfNeeded();
return appClientDelegate.getProgress();
@@ -1031,34 +1050,72 @@ public class YarnTaskSchedulerService extends TaskSchedulerService
}
return false;
}
+
+ private int scaleDownByPreemptionPercentage(int original) {
+ return (original + (preemptionPercentage - 1)) / preemptionPercentage;
+ }
void preemptIfNeeded() {
- ContainerId preemptedContainer = null;
+ if (preemptionPercentage == 0) {
+ // turned off
+ return;
+ }
+ ContainerId[] preemptedContainers = null;
+ int numPendingRequestsToService = 0;
synchronized (this) {
- Resource freeResources = Resources.subtract(totalResources,
- allocatedResources);
+ Resource freeResources = amRmClient.getAvailableResources();
if (LOG.isDebugEnabled()) {
LOG.debug("Allocated resource memory: " + allocatedResources.getMemory() +
" cpu:" + allocatedResources.getVirtualCores() +
- " delayedContainers: " + delayedContainerManager.delayedContainers.size());
+ " delayedContainers: " + delayedContainerManager.delayedContainers.size() +
+ " heartbeats: " + numHeartbeats + " lastPreemptionHeartbeat: " + heartbeatAtLastPreemption);
}
assert freeResources.getMemory() >= 0;
CookieContainerRequest highestPriRequest = null;
+ int numHighestPriRequests = 0;
for(CookieContainerRequest request : taskRequests.values()) {
if(highestPriRequest == null) {
highestPriRequest = request;
+ numHighestPriRequests = 1;
} else if(isHigherPriority(request.getPriority(),
highestPriRequest.getPriority())){
highestPriRequest = request;
+ numHighestPriRequests = 1;
+ } else if (request.getPriority().equals(highestPriRequest.getPriority())) {
+ numHighestPriRequests++;
}
}
- if(highestPriRequest != null &&
- !fitsIn(highestPriRequest.getCapability(), freeResources)) {
- // highest priority request will not fit in existing free resources
- // free up some more
- // TODO this is subject to error wrt RM resource normalization
-
+
+ if (highestPriRequest == null) {
+ // nothing pending
+ return;
+ }
+
+ if(fitsIn(highestPriRequest.getCapability(), freeResources)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Highest pri request: " + highestPriRequest + " fits in available resources "
+ + freeResources);
+ }
+ return;
+ }
+ // highest priority request will not fit in existing free resources
+ // free up some more
+ // TODO this is subject to error wrt RM resource normalization
+
+ numPendingRequestsToService = scaleDownByPreemptionPercentage(numHighestPriRequests);
+
+ if (numPendingRequestsToService < 1) {
+ return;
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Trying to service " + numPendingRequestsToService + " out of total "
+ + numHighestPriRequests + " pending requests at pri: "
+ + highestPriRequest.getPriority());
+ }
+
+ for (int i=0; i<numPendingRequestsToService; ++i) {
// This request must have been considered for matching with all existing
// containers when request was made.
Container lowestPriNewContainer = null;
@@ -1093,6 +1150,7 @@ public class YarnTaskSchedulerService extends TaskSchedulerService
" with priority: " + lowestPriNewContainer.getPriority() +
" to free resource for request: " + highestPriRequest +
" . Current free resources: " + freeResources);
+ numPendingRequestsToService--;
releaseUnassignedContainers(Collections.singletonList(lowestPriNewContainer));
// We are returning an unused resource back the RM. The RM thinks it
// has serviced our initial request and will not re-allocate this back
@@ -1116,54 +1174,91 @@ public class YarnTaskSchedulerService extends TaskSchedulerService
break;
}
}
-
- return;
+ // come back and free more new containers if needed
+ continue;
}
+ }
+
+ if (numPendingRequestsToService < 1) {
+ return;
+ }
+
+ // there are no reused or new containers to release. try to preempt running containers
+ // this assert will be a no-op in production but can help identify
+ // invalid assumptions during testing
+ assert delayedContainerManager.delayedContainers.isEmpty();
+
+ if ((numHeartbeats - heartbeatAtLastPreemption) <= numHeartbeatsBetweenPreemptions) {
+ return;
+ }
- // this assert will be a no-op in production but can help identify
- // invalid assumptions during testing
- assert delayedContainerManager.delayedContainers.isEmpty();
-
- // there are no reused or new containers to release
- // try to preempt running containers
- Map.Entry<Object, Container> preemptedEntry = null;
- for(Map.Entry<Object, Container> entry : taskAllocations.entrySet()) {
- HeldContainer heldContainer = heldContainers.get(entry.getValue().getId());
- CookieContainerRequest lastTaskInfo = heldContainer.getLastTaskInfo();
- Priority taskPriority = lastTaskInfo.getPriority();
- Object signature = lastTaskInfo.getCookie().getContainerSignature();
- if(!isHigherPriority(highestPriRequest.getPriority(), taskPriority)) {
- // higher or same priority
- continue;
- }
- if (containerSignatureMatcher.isExactMatch(
- highestPriRequest.getCookie().getContainerSignature(),
- signature)) {
- // exact match with different priorities
- continue;
- }
- if(preemptedEntry == null ||
- !isHigherPriority(taskPriority,
- preemptedEntry.getValue().getPriority())) {
- // keep the lower priority or the one added later
- preemptedEntry = entry;
+ Priority preemptedTaskPriority = null;
+ int numEntriesAtPreemptedPriority = 0;
+ for(Map.Entry<Object, Container> entry : taskAllocations.entrySet()) {
+ HeldContainer heldContainer = heldContainers.get(entry.getValue().getId());
+ CookieContainerRequest lastTaskInfo = heldContainer.getLastTaskInfo();
+ Priority taskPriority = lastTaskInfo.getPriority();
+ Object signature = lastTaskInfo.getCookie().getContainerSignature();
+ if(!isHigherPriority(highestPriRequest.getPriority(), taskPriority)) {
+ // higher or same priority
+ continue;
+ }
+ if (containerSignatureMatcher.isExactMatch(
+ highestPriRequest.getCookie().getContainerSignature(),
+ signature)) {
+ // exact match with different priorities
+ continue;
+ }
+ if (preemptedTaskPriority == null ||
+ !isHigherPriority(taskPriority, preemptedTaskPriority)) {
+ // keep the lower priority
+ preemptedTaskPriority = taskPriority;
+ if (taskPriority.equals(preemptedTaskPriority)) {
+ numEntriesAtPreemptedPriority++;
+ } else {
+ // this is at a lower priority than existing
+ numEntriesAtPreemptedPriority = 1;
}
}
- if(preemptedEntry != null) {
- // found something to preempt
- LOG.info("Preempting task: " + preemptedEntry.getKey() +
- " to free resource for request: " + highestPriRequest +
- " . Current free resources: " + freeResources);
- preemptedContainer = preemptedEntry.getValue().getId();
- // app client will be notified when after container is killed
- // and we get its completed container status
+ }
+ if(preemptedTaskPriority != null) {
+ int newNumPendingRequestsToService = scaleDownByPreemptionPercentage(Math.min(
+ numEntriesAtPreemptedPriority, numHighestPriRequests));
+ numPendingRequestsToService = Math.min(newNumPendingRequestsToService,
+ numPendingRequestsToService);
+ if (numPendingRequestsToService < 1) {
+ return;
+ }
+ LOG.info("Trying to service " + numPendingRequestsToService + " out of total "
+ + numHighestPriRequests + " pending requests at pri: "
+ + highestPriRequest.getPriority() + " by preempting from "
+ + numEntriesAtPreemptedPriority + " running tasks at priority: " + preemptedTaskPriority);
+ // found something to preempt. get others of the same priority
+ preemptedContainers = new ContainerId[numPendingRequestsToService];
+ int currIndex = 0;
+ for (Map.Entry<Object, Container> entry : taskAllocations.entrySet()) {
+ Container container = entry.getValue();
+ if (preemptedTaskPriority.equals(container.getPriority())) {
+ // taskAllocations map will iterate from oldest to newest assigned containers
+ // keep the N newest containersIds with the matching priority
+ preemptedContainers[currIndex++ % numPendingRequestsToService] = container.getId();
+ }
}
+ // app client will be notified when after container is killed
+ // and we get its completed container status
}
}
// upcall outside locks
- if (preemptedContainer != null) {
- appClientDelegate.preemptContainer(preemptedContainer);
+ if (preemptedContainers != null) {
+ heartbeatAtLastPreemption = numHeartbeats;
+ for(int i=0; i<numPendingRequestsToService; ++i) {
+ ContainerId cId = preemptedContainers[i];
+ if (cId != null) {
+ LOG.info("Preempting container: " + cId + " currently allocated to a task.");
+ appClientDelegate.preemptContainer(cId);
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/5df90046/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
index bd53f87..3afff7c 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
@@ -1202,7 +1202,7 @@ public class TestTaskScheduler {
}
@SuppressWarnings({ "unchecked", "rawtypes" })
- @Test
+ @Test (timeout=5000)
public void testTaskSchedulerPreemption() throws Exception {
RackResolver.init(new YarnConfiguration());
TaskSchedulerAppCallback mockApp = mock(TaskSchedulerAppCallback.class);
@@ -1248,7 +1248,8 @@ public class TestTaskScheduler {
Object mockTask3 = mock(Object.class);
Object mockTask3Wait = mock(Object.class);
Object mockTask3Retry = mock(Object.class);
- Object mockTask3Kill = mock(Object.class);
+ Object mockTask3KillA = mock(Object.class);
+ Object mockTask3KillB = mock(Object.class);
Object obj3 = new Object();
Priority pri2 = Priority.newInstance(2);
Priority pri4 = Priority.newInstance(4);
@@ -1275,12 +1276,19 @@ public class TestTaskScheduler {
addContainerRequest(requestCaptor.capture());
anyContainers.add(requestCaptor.getValue());
// later one in the allocation gets killed between the two task3's
- scheduler.allocateTask(mockTask3Kill, taskAsk, null,
+ scheduler.allocateTask(mockTask3KillA, taskAsk, null,
null, pri6, obj3, null);
drainableAppCallback.drain();
verify(mockRMClient, times(3)).
addContainerRequest(requestCaptor.capture());
anyContainers.add(requestCaptor.getValue());
+ // later one in the allocation gets killed between the two task3's
+ scheduler.allocateTask(mockTask3KillB, taskAsk, null,
+ null, pri6, obj3, null);
+ drainableAppCallback.drain();
+ verify(mockRMClient, times(4)).
+ addContainerRequest(requestCaptor.capture());
+ anyContainers.add(requestCaptor.getValue());
Resource freeResource = Resource.newInstance(500, 0);
when(mockRMClient.getAvailableResources()).thenReturn(freeResource);
@@ -1310,13 +1318,20 @@ public class TestTaskScheduler {
ContainerId mockCId2 = mock(ContainerId.class);
when(mockContainer2.getId()).thenReturn(mockCId2);
containers.add(mockContainer2);
- Container mockContainer3 = mock(Container.class, RETURNS_DEEP_STUBS);
- when(mockContainer3.getNodeId().getHost()).thenReturn("host1");
- when(mockContainer3.getResource()).thenReturn(taskAsk);
- when(mockContainer3.getPriority()).thenReturn(pri6);
- ContainerId mockCId3 = mock(ContainerId.class);
- when(mockContainer3.getId()).thenReturn(mockCId3);
- containers.add(mockContainer3);
+ Container mockContainer3A = mock(Container.class, RETURNS_DEEP_STUBS);
+ when(mockContainer3A.getNodeId().getHost()).thenReturn("host1");
+ when(mockContainer3A.getResource()).thenReturn(taskAsk);
+ when(mockContainer3A.getPriority()).thenReturn(pri6);
+ ContainerId mockCId3A = mock(ContainerId.class);
+ when(mockContainer3A.getId()).thenReturn(mockCId3A);
+ containers.add(mockContainer3A);
+ Container mockContainer3B = mock(Container.class, RETURNS_DEEP_STUBS);
+ when(mockContainer3B.getNodeId().getHost()).thenReturn("host1");
+ when(mockContainer3B.getResource()).thenReturn(taskAsk);
+ when(mockContainer3B.getPriority()).thenReturn(pri6);
+ ContainerId mockCId3B = mock(ContainerId.class);
+ when(mockContainer3B.getId()).thenReturn(mockCId3B);
+ containers.add(mockContainer3B);
when(
mockRMClient.getMatchingRequests((Priority) any(), eq("host1"),
(Resource) any())).thenAnswer(
@@ -1368,14 +1383,16 @@ public class TestTaskScheduler {
scheduler.onContainersAllocated(containers);
drainableAppCallback.drain();
- Assert.assertEquals(3, scheduler.taskAllocations.size());
- Assert.assertEquals(3072, scheduler.allocatedResources.getMemory());
+ Assert.assertEquals(4, scheduler.taskAllocations.size());
+ Assert.assertEquals(4096, scheduler.allocatedResources.getMemory());
Assert.assertEquals(mockCId1,
scheduler.taskAllocations.get(mockTask1).getId());
Assert.assertEquals(mockCId2,
scheduler.taskAllocations.get(mockTask3).getId());
- Assert.assertEquals(mockCId3,
- scheduler.taskAllocations.get(mockTask3Kill).getId());
+ Assert.assertEquals(mockCId3A,
+ scheduler.taskAllocations.get(mockTask3KillA).getId());
+ Assert.assertEquals(mockCId3B,
+ scheduler.taskAllocations.get(mockTask3KillB).getId());
// no preemption
scheduler.getProgress();
@@ -1419,7 +1436,7 @@ public class TestTaskScheduler {
drainableAppCallback.drain();
verify(mockRMClient, times(1)).releaseAssignedContainer((ContainerId)any());
verify(mockRMClient, times(1)).releaseAssignedContainer(mockCId4);
- verify(mockRMClient, times(4)).
+ verify(mockRMClient, times(5)).
addContainerRequest(requestCaptor.capture());
CookieContainerRequest reAdded = requestCaptor.getValue();
Assert.assertEquals(pri6, reAdded.getPriority());
@@ -1436,15 +1453,27 @@ public class TestTaskScheduler {
drainableAppCallback.drain();
verify(mockRMClient, times(1)).releaseAssignedContainer((ContainerId)any());
- scheduler.allocateTask(mockTask2, taskAsk, null,
- null, pri4, null, null);
+ for (int i=0; i<11; ++i) {
+ scheduler.allocateTask(mockTask2, taskAsk, null,
+ null, pri4, null, null);
+ }
drainableAppCallback.drain();
- // mockTaskPri3Kill gets preempted
+ // mockTaskPri3KillB gets preempted to clear 10% of outstanding running preemptable tasks
scheduler.getProgress();
drainableAppCallback.drain();
verify(mockRMClient, times(2)).releaseAssignedContainer((ContainerId)any());
- verify(mockRMClient, times(1)).releaseAssignedContainer(mockCId3);
+ verify(mockRMClient, times(1)).releaseAssignedContainer(mockCId3B);
+ // next 3 heartbeats do nothing, waiting for the RM to act on the last released resources
+ scheduler.getProgress();
+ scheduler.getProgress();
+ scheduler.getProgress();
+ verify(mockRMClient, times(2)).releaseAssignedContainer((ContainerId)any());
+ scheduler.getProgress();
+ drainableAppCallback.drain();
+ // Next oldest mockTaskPri3KillA gets preempted to clear 10% of outstanding running preemptable tasks
+ verify(mockRMClient, times(3)).releaseAssignedContainer((ContainerId)any());
+ verify(mockRMClient, times(1)).releaseAssignedContainer(mockCId3A);
AppFinalStatus finalStatus =
new AppFinalStatus(FinalApplicationStatus.SUCCEEDED, "", appUrl);
[06/50] [abbrv] tez git commit: TEZ-1701. ATS fixes to flush all
history events and also using batching. (hitesh)
Posted by je...@apache.org.
TEZ-1701. ATS fixes to flush all history events and also using batching. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/a6c8006c
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/a6c8006c
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/a6c8006c
Branch: refs/heads/TEZ-8
Commit: a6c8006c37334099a8a77befa3b687113514a35c
Parents: 74c7f87
Author: Hitesh Shah <hi...@apache.org>
Authored: Thu Oct 30 07:54:45 2014 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Thu Oct 30 07:54:45 2014 -0700
----------------------------------------------------------------------
.../apache/tez/dag/api/TezConfiguration.java | 19 ++-
.../logging/ats/ATSHistoryLoggingService.java | 131 +++++++++++++------
.../ats/TestATSHistoryLoggingService.java | 22 +++-
3 files changed, 126 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/a6c8006c/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 1919f5c..7cc0aa5 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -733,7 +733,24 @@ public class TezConfiguration extends Configuration {
public static final String YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS =
TEZ_PREFIX + "yarn.ats.event.flush.timeout.millis";
public static final long YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS_DEFAULT =
- 3000l;
+ -1;
+
+ /**
+ * Int value. Max no. of events to send in a single batch to ATS.
+ * Expert level setting.
+ */
+ public static final String YARN_ATS_MAX_EVENTS_PER_BATCH =
+ TEZ_PREFIX + "yarn.ats.max.events.per.batch";
+ public static final int YARN_ATS_MAX_EVENTS_PER_BATCH_DEFAULT = 5;
+
+
+ /**
+ * Int value. Time, in milliseconds, to wait for an event before sending a batch to ATS.
+ * Expert level setting.
+ */
+ public static final String YARN_ATS_MAX_POLLING_TIME_PER_EVENT = TEZ_PREFIX
+ + "yarn.ats.max.polling.time.per.event.millis";
+ public static final int YARN_ATS_MAX_POLLING_TIME_PER_EVENT_DEFAULT = 10;
/**
* Boolean value. Enable recovery of DAGs. This allows a restarted app master to recover the
http://git-wip-us.apache.org/repos/asf/tez/blob/a6c8006c/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java
index aa1dd06..cdfbf7c 100644
--- a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java
+++ b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java
@@ -19,12 +19,16 @@
package org.apache.tez.dag.history.logging.ats;
import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError;
import org.apache.hadoop.yarn.client.api.TimelineClient;
@@ -56,6 +60,10 @@ public class ATSHistoryLoggingService extends HistoryLoggingService {
private HashSet<TezDAGID> skippedDAGs = new HashSet<TezDAGID>();
private long maxTimeToWaitOnShutdown;
+ private boolean waitForeverOnShutdown = false;
+
+ private int maxEventsPerBatch;
+ private long maxPollingTimeMillis;
public ATSHistoryLoggingService() {
super(ATSHistoryLoggingService.class.getName());
@@ -69,17 +77,29 @@ public class ATSHistoryLoggingService extends HistoryLoggingService {
maxTimeToWaitOnShutdown = conf.getLong(
TezConfiguration.YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS,
TezConfiguration.YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS_DEFAULT);
+ maxEventsPerBatch = conf.getInt(
+ TezConfiguration.YARN_ATS_MAX_EVENTS_PER_BATCH,
+ TezConfiguration.YARN_ATS_MAX_EVENTS_PER_BATCH_DEFAULT);
+ maxPollingTimeMillis = conf.getInt(
+ TezConfiguration.YARN_ATS_MAX_POLLING_TIME_PER_EVENT,
+ TezConfiguration.YARN_ATS_MAX_POLLING_TIME_PER_EVENT_DEFAULT);
+ if (maxTimeToWaitOnShutdown < 0) {
+ waitForeverOnShutdown = true;
+ }
}
@Override
public void serviceStart() {
LOG.info("Starting ATSService");
timelineClient.start();
+
eventHandlingThread = new Thread(new Runnable() {
@Override
public void run() {
- DAGHistoryEvent event;
- while (!stopped.get() && !Thread.currentThread().isInterrupted()) {
+ List<DAGHistoryEvent> events = new LinkedList<DAGHistoryEvent>();
+ boolean interrupted = false;
+ while (!stopped.get() && !Thread.currentThread().isInterrupted()
+ && !interrupted) {
// Log the size of the event-queue every so often.
if (eventCounter != 0 && eventCounter % 1000 == 0) {
@@ -92,20 +112,18 @@ public class ATSHistoryLoggingService extends HistoryLoggingService {
++eventCounter;
}
- try {
- event = eventQueue.take();
- } catch (InterruptedException e) {
- LOG.info("EventQueue take interrupted. Returning");
- return;
- }
-
synchronized (lock) {
- ++eventsProcessed;
try {
- handleEvent(event);
+ getEventBatch(events);
+ } catch (InterruptedException e) {
+ // Finish processing events and then return
+ interrupted = true;
+ }
+ eventsProcessed += events.size();
+ try {
+ handleEvents(events);
} catch (Exception e) {
- // TODO handle failures - treat as fatal or ignore?
- LOG.warn("Error handling event", e);
+ LOG.warn("Error handling events", e);
}
}
}
@@ -126,21 +144,27 @@ public class ATSHistoryLoggingService extends HistoryLoggingService {
if (!eventQueue.isEmpty()) {
LOG.warn("ATSService being stopped"
+ ", eventQueueBacklog=" + eventQueue.size()
- + ", maxTimeLeftToFlush=" + maxTimeToWaitOnShutdown);
+ + ", maxTimeLeftToFlush=" + maxTimeToWaitOnShutdown
+ + ", waitForever=" + waitForeverOnShutdown);
long startTime = appContext.getClock().getTime();
- if (maxTimeToWaitOnShutdown > 0) {
- long endTime = startTime + maxTimeToWaitOnShutdown;
- while (endTime >= appContext.getClock().getTime()) {
- DAGHistoryEvent event = eventQueue.poll();
- if (event == null) {
- break;
- }
- try {
- handleEvent(event);
- } catch (Exception e) {
- LOG.warn("Error handling event", e);
- break;
- }
+ long endTime = startTime + maxTimeToWaitOnShutdown;
+ List<DAGHistoryEvent> events = new LinkedList<DAGHistoryEvent>();
+ while (waitForeverOnShutdown || (endTime >= appContext.getClock().getTime())) {
+ try {
+ getEventBatch(events);
+ } catch (InterruptedException e) {
+ LOG.info("ATSService interrupted while shutting down. Exiting."
+ + " EventQueueBacklog=" + eventQueue.size());
+ }
+ if (events.isEmpty()) {
+ LOG.info("Event queue empty, stopping ATS Service");
+ break;
+ }
+ try {
+ handleEvents(events);
+ } catch (Exception e) {
+ LOG.warn("Error handling event", e);
+ break;
}
}
}
@@ -152,13 +176,33 @@ public class ATSHistoryLoggingService extends HistoryLoggingService {
timelineClient.stop();
}
+ private void getEventBatch(List<DAGHistoryEvent> events) throws InterruptedException {
+ events.clear();
+ int counter = 0;
+ while (counter < maxEventsPerBatch) {
+ DAGHistoryEvent event = eventQueue.poll(maxPollingTimeMillis, TimeUnit.MILLISECONDS);
+ if (event == null) {
+ break;
+ }
+ if (!isValidEvent(event)) {
+ continue;
+ }
+ ++counter;
+ events.add(event);
+ if (event.getHistoryEvent().getEventType().equals(HistoryEventType.DAG_SUBMITTED)) {
+ // Special case this as it might be a large payload
+ break;
+ }
+ }
+ }
+
+
public void handle(DAGHistoryEvent event) {
eventQueue.add(event);
}
- private void handleEvent(DAGHistoryEvent event) {
+ private boolean isValidEvent(DAGHistoryEvent event) {
HistoryEventType eventType = event.getHistoryEvent().getEventType();
-
TezDAGID dagId = event.getDagID();
if (eventType.equals(HistoryEventType.DAG_SUBMITTED)) {
@@ -167,43 +211,52 @@ public class ATSHistoryLoggingService extends HistoryLoggingService {
String dagName = dagSubmittedEvent.getDAGName();
if (dagName != null
&& dagName.startsWith(
- TezConstants.TEZ_PREWARM_DAG_NAME_PREFIX)) {
+ TezConstants.TEZ_PREWARM_DAG_NAME_PREFIX)) {
// Skip recording pre-warm DAG events
skippedDAGs.add(dagId);
- return;
+ return false;
}
}
if (eventType.equals(HistoryEventType.DAG_FINISHED)) {
// Remove from set to keep size small
// No more events should be seen after this point.
if (skippedDAGs.remove(dagId)) {
- return;
+ return false;
}
}
if (dagId != null && skippedDAGs.contains(dagId)) {
// Skip pre-warm DAGs
- return;
+ return false;
+ }
+
+ return true;
+ }
+
+
+
+ private void handleEvents(List<DAGHistoryEvent> events) {
+ TimelineEntity[] entities = new TimelineEntity[events.size()];
+ for (int i = 0; i < events.size(); ++i) {
+ entities[i] = HistoryEventTimelineConversion.convertToTimelineEntity(
+ events.get(i).getHistoryEvent());
}
try {
TimelinePutResponse response =
- timelineClient.putEntities(
- HistoryEventTimelineConversion.convertToTimelineEntity(event.getHistoryEvent()));
+ timelineClient.putEntities(entities);
if (response != null
&& !response.getErrors().isEmpty()) {
TimelinePutError err = response.getErrors().get(0);
if (err.getErrorCode() != 0) {
- LOG.warn("Could not post history event to ATS, eventType="
- + eventType
+ LOG.warn("Could not post history events to ATS"
+ ", atsPutError=" + err.getErrorCode());
}
}
// Do nothing additional, ATS client library should handle throttling
// or auto-disable as needed
} catch (Exception e) {
- LOG.warn("Could not handle history event, eventType="
- + eventType, e);
+ LOG.warn("Could not handle history events", e);
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/a6c8006c/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryLoggingService.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryLoggingService.java b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryLoggingService.java
index 7bd3b91..06bc065 100644
--- a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryLoggingService.java
+++ b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryLoggingService.java
@@ -32,10 +32,10 @@ import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.Matchers;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
-import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -45,6 +45,7 @@ public class TestATSHistoryLoggingService {
private AppContext appContext;
private Configuration conf;
private int atsInvokeCounter;
+ private int atsEntitiesCounter;
private SystemClock clock = new SystemClock();
@Before
@@ -56,14 +57,17 @@ public class TestATSHistoryLoggingService {
conf.setLong(TezConfiguration.YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS,
1000l);
atsInvokeCounter = 0;
+ atsEntitiesCounter = 0;
atsHistoryLoggingService.init(conf);
atsHistoryLoggingService.timelineClient = mock(TimelineClient.class);
atsHistoryLoggingService.start();
when(appContext.getClock()).thenReturn(clock);
- when(atsHistoryLoggingService.timelineClient.putEntities(any(TimelineEntity.class))).thenAnswer(
+ when(atsHistoryLoggingService.timelineClient.putEntities(
+ Matchers.<TimelineEntity[]>anyVararg())).thenAnswer(
new Answer<Object>() {
@Override
public Object answer(InvocationOnMock invocation) throws Throwable {
+ atsEntitiesCounter += invocation.getArguments().length;
++atsInvokeCounter;
try {
Thread.sleep(500);
@@ -89,20 +93,26 @@ public class TestATSHistoryLoggingService {
DAGHistoryEvent historyEvent = new DAGHistoryEvent(tezDAGID,
new DAGStartedEvent(tezDAGID, 1001l, "user1", "dagName1"));
- for (int i = 0; i < 20; ++i) {
+ for (int i = 0; i < 100; ++i) {
atsHistoryLoggingService.handle(historyEvent);
}
try {
- Thread.sleep(2500l);
+ Thread.sleep(500l);
} catch (InterruptedException e) {
// Do nothing
}
atsHistoryLoggingService.stop();
- Assert.assertTrue(atsInvokeCounter >= 4);
- Assert.assertTrue(atsInvokeCounter < 10);
+ Assert.assertTrue("ATSEntitiesCounter = " + atsEntitiesCounter, atsEntitiesCounter >= 10);
+ Assert.assertTrue("ATSEntitiesCounter = " + atsEntitiesCounter, atsEntitiesCounter < 50);
+ Assert.assertTrue("ATSInvokeCounter = " + atsInvokeCounter, atsInvokeCounter >= 1);
+ Assert.assertTrue("ATSInvokeCounter = " + atsInvokeCounter, atsInvokeCounter <= 10);
+ Assert.assertTrue("ATSInvokeCounter = " + atsInvokeCounter
+ + ", ATSEntitiesCounter = " + atsEntitiesCounter,
+ atsEntitiesCounter >=
+ ((atsInvokeCounter-1) * TezConfiguration.YARN_ATS_MAX_EVENTS_PER_BATCH_DEFAULT));
}
}
[41/50] [abbrv] tez git commit: TEZ-1687. Use logIdentifier of Vertex
for logging (zjffdu)
Posted by je...@apache.org.
TEZ-1687. Use logIdentifier of Vertex for logging (zjffdu)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/103839b6
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/103839b6
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/103839b6
Branch: refs/heads/TEZ-8
Commit: 103839b6ccbee0de646bfd3eaf25b8f5ec73a942
Parents: 0af0213
Author: Jeff Zhang <zj...@apache.org>
Authored: Tue Nov 11 12:34:22 2014 +0800
Committer: Jeff Zhang <zj...@apache.org>
Committed: Tue Nov 11 12:34:22 2014 +0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/tez/dag/app/dag/impl/DAGImpl.java | 28 +++++-----
.../org/apache/tez/dag/app/dag/impl/Edge.java | 14 ++---
.../apache/tez/dag/app/dag/impl/VertexImpl.java | 54 ++++++++++----------
.../tez/dag/app/dag/impl/VertexManager.java | 4 +-
5 files changed, 51 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/103839b6/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 57a0c7f..34e2c7a 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -21,6 +21,7 @@ ALL CHANGES:
TEZ-1749. Increase test timeout for TestLocalMode.testMultipleClientsWithSession
TEZ-1750. Add a DAGScheduler which schedules tasks only when sources have been scheduled.
TEZ-1761. TestRecoveryParser::testGetLastInProgressDAG fails in similar manner to TEZ-1686.
+ TEZ-1687. Use logIdentifier of Vertex for logging.
Release 0.5.2: 2014-11-07
http://git-wip-us.apache.org/repos/asf/tez/blob/103839b6/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index f877eb4..de62752 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -772,7 +772,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
for (Vertex v : vertices.values()) {
if (v.getInputVerticesCount() == 0) {
if (LOG.isDebugEnabled()) {
- LOG.debug("Initing root vertex " + v.getName());
+ LOG.debug("Initing root vertex " + v.getLogIdentifier());
}
eventHandler.handle(new VertexEvent(v.getVertexId(),
VertexEventType.V_INIT));
@@ -781,7 +781,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
for (Vertex v : vertices.values()) {
if (v.getInputVerticesCount() == 0) {
if (LOG.isDebugEnabled()) {
- LOG.debug("Starting root vertex " + v.getName());
+ LOG.debug("Starting root vertex " + v.getLogIdentifier());
}
eventHandler.handle(new VertexEvent(v.getVertexId(),
VertexEventType.V_START));
@@ -851,7 +851,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
break;
}
if (vertex.getOutputCommitters() == null) {
- LOG.info("No output committers for vertex: " + vertex.getName());
+ LOG.info("No output committers for vertex: " + vertex.getLogIdentifier());
continue;
}
Map<String, OutputCommitter> outputCommitters =
@@ -868,14 +868,14 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
}
}
if (outputCommitters.isEmpty()) {
- LOG.info("No exclusive output committers for vertex: " + vertex.getName());
+ LOG.info("No exclusive output committers for vertex: " + vertex.getLogIdentifier());
continue;
}
for (Map.Entry<String, OutputCommitter> entry : outputCommitters.entrySet()) {
LOG.info("Committing output: " + entry.getKey() + " for vertex: "
- + vertex.getVertexId());
+ + vertex.getLogIdentifier());
if (vertex.getState() != VertexState.SUCCEEDED) {
- throw new TezUncheckedException("Vertex: " + vertex.getName() +
+ throw new TezUncheckedException("Vertex: " + vertex.getLogIdentifier() +
" not in SUCCEEDED state. State= " + vertex.getState());
}
if (!commitOutput(entry.getKey(), entry.getValue())) {
@@ -897,7 +897,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
Map<String, OutputCommitter> outputCommitters = vertex
.getOutputCommitters();
if (outputCommitters == null || outputCommitters.isEmpty()) {
- LOG.info("No output committers for vertex: " + vertex.getName());
+ LOG.info("No output committers for vertex: " + vertex.getLogIdentifier());
continue;
}
for (Map.Entry<String, OutputCommitter> entry : outputCommitters
@@ -907,7 +907,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
|| vertex.getState() != VertexState.SUCCEEDED // never commit unsuccessful outputs
) {
LOG.info("Aborting output: " + entry.getKey() + " for vertex: "
- + vertex.getVertexId());
+ + vertex.getLogIdentifier());
try {
getDagUGI().doAs(new PrivilegedExceptionAction<Void>() {
@Override
@@ -918,7 +918,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
});
} catch (Exception e) {
LOG.info("Exception in aborting output: " + entry.getKey()
- + " for vertex: " + vertex.getVertexId(), e);
+ + " for vertex: " + vertex.getLogIdentifier(), e);
}
}
// else successful outputs have already been committed
@@ -1422,7 +1422,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
if (v.getInputVerticesCount() == 0) {
if (LOG.isDebugEnabled()) {
LOG.debug("Sending Running Recovery event to root vertex "
- + v.getName());
+ + v.getLogIdentifier());
}
dag.eventHandler.handle(new VertexEventRecoverVertex(v.getVertexId(),
VertexState.RUNNING));
@@ -1478,7 +1478,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
if (v.getInputVerticesCount() == 0) {
if (LOG.isDebugEnabled()) {
LOG.debug("Sending Running Recovery event to root vertex "
- + v.getName());
+ + v.getLogIdentifier());
}
dag.eventHandler.handle(new VertexEventRecoverVertex(v.getVertexId(),
VertexState.RUNNING));
@@ -1702,7 +1702,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
job.reRunningVertices.remove(vertex.getVertexId());
- LOG.info("Vertex " + vertex.getVertexId() + " completed."
+ LOG.info("Vertex " + vertex.getLogIdentifier() + " completed."
+ ", numCompletedVertices=" + job.numCompletedVertices
+ ", numSuccessfulVertices=" + job.numSuccessfulVertices
+ ", numFailedVertices=" + job.numFailedVertices
@@ -1732,7 +1732,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
boolean failed = job.vertexReRunning(vertex);
- LOG.info("Vertex " + vertex.getVertexId() + " re-running."
+ LOG.info("Vertex " + vertex.getLogIdentifier() + " re-running."
+ ", numCompletedVertices=" + job.numCompletedVertices
+ ", numSuccessfulVertices=" + job.numSuccessfulVertices
+ ", numFailedVertices=" + job.numFailedVertices
@@ -1839,7 +1839,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
for (VertexGroupInfo groupInfo : groupList) {
if (groupInfo.committed) {
LOG.info("Aborting job as committed vertex: "
- + vertex.getVertexId() + " is re-running");
+ + vertex.getLogIdentifier() + " is re-running");
enactKill(DAGTerminationCause.COMMIT_FAILURE,
VertexTerminationCause.COMMIT_FAILURE);
return true;
http://git-wip-us.apache.org/repos/asf/tez/blob/103839b6/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/Edge.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/Edge.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/Edge.java
index 360a839..05dfc6e 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/Edge.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/Edge.java
@@ -190,7 +190,7 @@ public class Edge {
public void setSourceVertex(Vertex sourceVertex) {
if (this.sourceVertex != null && this.sourceVertex != sourceVertex) {
throw new TezUncheckedException("Source vertex exists: "
- + sourceVertex.getName());
+ + sourceVertex.getLogIdentifier());
}
this.sourceVertex = sourceVertex;
}
@@ -199,7 +199,7 @@ public class Edge {
if (this.destinationVertex != null
&& this.destinationVertex != destinationVertex) {
throw new TezUncheckedException("Destination vertex exists: "
- + destinationVertex.getName());
+ + destinationVertex.getLogIdentifier());
}
this.destinationVertex = destinationVertex;
}
@@ -277,7 +277,7 @@ public class Edge {
Task srcTask = sourceVertex.getTask(srcTaskIndex);
if (srcTask == null) {
throw new TezUncheckedException("Unexpected null task." +
- " sourceVertex=" + sourceVertex.getVertexId() +
+ " sourceVertex=" + sourceVertex.getLogIdentifier() +
" srcIndex = " + srcTaskIndex +
" destAttemptId=" + destAttemptId +
" destIndex=" + destTaskIndex +
@@ -355,9 +355,9 @@ public class Edge {
Task destTask = destinationVertex.getTask(destTaskIndex);
if (destTask == null) {
throw new TezUncheckedException("Unexpected null task." +
- " sourceVertex=" + sourceVertex.getVertexId() +
+ " sourceVertex=" + sourceVertex.getLogIdentifier() +
" srcTaskIndex = " + srcTaskIndex +
- " destVertex=" + destinationVertex.getVertexId() +
+ " destVertex=" + destinationVertex.getLogIdentifier() +
" destTaskIndex=" + destTaskIndex +
" destNumTasks=" + destinationVertex.getTotalTasks() +
" edgeManager=" + edgeManager.getClass().getName());
@@ -453,9 +453,9 @@ public class Edge {
private String generateCommonDebugString(int srcTaskIndex, TezEvent tezEvent) {
return new StringBuilder()
- .append(" sourceVertex=").append(sourceVertex.getVertexId())
+ .append(" sourceVertex=").append(sourceVertex.getLogIdentifier())
.append(" srcIndex = ").append(srcTaskIndex)
- .append(" destAttemptId=").append(destinationVertex.getVertexId())
+ .append(" destAttemptId=").append(destinationVertex.getLogIdentifier())
.append(" edgeManager=").append(edgeManager.getClass().getName())
.append(" Event type=").append(tezEvent.getEventType()).toString();
}
http://git-wip-us.apache.org/repos/asf/tez/blob/103839b6/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index 593ecca..8130c18 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -1251,7 +1251,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
for(Map.Entry<String, EdgeManagerPluginDescriptor> entry :
sourceEdgeManagers.entrySet()) {
LOG.info("Recovering edge manager for source:"
- + entry.getKey() + " destination: " + getVertexId());
+ + entry.getKey() + " destination: " + getLogIdentifier());
Vertex sourceVertex = appContext.getCurrentDAG().getVertex(entry.getKey());
Edge edge = sourceVertices.get(sourceVertex);
try {
@@ -1309,7 +1309,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
if(sourceEdgeManagers != null) {
for(Map.Entry<String, EdgeManagerPluginDescriptor> entry : sourceEdgeManagers.entrySet()) {
LOG.info("Replacing edge manager for source:"
- + entry.getKey() + " destination: " + getVertexId());
+ + entry.getKey() + " destination: " + getLogIdentifier());
Vertex sourceVertex = appContext.getCurrentDAG().getVertex(entry.getKey());
Edge edge = sourceVertices.get(sourceVertex);
try {
@@ -1341,7 +1341,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
stateChangeNotifier.stateChanged(vertexId,
new VertexStateUpdateParallelismUpdated(vertexName, numTasks, oldNumTasks));
this.createTasks();
- LOG.info("Vertex " + getVertexId() +
+ LOG.info("Vertex " + getLogIdentifier() +
" parallelism set to " + parallelism);
if (canInitVertex()) {
getEventHandler().handle(new VertexEvent(getVertexId(), VertexEventType.V_READY_TO_INIT));
@@ -1389,7 +1389,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
Task task = entry.getValue();
if (task.getState() != TaskState.NEW) {
String msg = "All tasks must be in initial state when changing parallelism"
- + " for vertex: " + getVertexId() + " name: " + getName();
+ + " for vertex: " + getLogIdentifier();
LOG.warn(msg);
throw new TezUncheckedException(msg);
}
@@ -1411,7 +1411,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
if(sourceEdgeManagers != null) {
for(Map.Entry<String, EdgeManagerPluginDescriptor> entry : sourceEdgeManagers.entrySet()) {
LOG.info("Replacing edge manager for source:"
- + entry.getKey() + " destination: " + getVertexId());
+ + entry.getKey() + " destination: " + getLogIdentifier());
Vertex sourceVertex = appContext.getCurrentDAG().getVertex(entry.getKey());
Edge edge = sourceVertices.get(sourceVertex);
try {
@@ -2117,7 +2117,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
.convertVertexManagerPluginDescriptorFromDAGPlan(vertexPlan
.getVertexManagerPlugin());
LOG.info("Setting user vertex manager plugin: "
- + pluginDesc.getClassName() + " on vertex: " + getName());
+ + pluginDesc.getClassName() + " on vertex: " + getLogIdentifier());
vertexManager = new VertexManager(pluginDesc, this, appContext, stateChangeNotifier);
} else {
// Intended order of picking a vertex manager
@@ -2453,14 +2453,14 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
if (destEdge == null) {
throw new TezUncheckedException("Bad destination vertex: " +
sourceMeta.getEdgeVertexName() + " for event vertex: " +
- getVertexId());
+ getLogIdentifier());
}
if (LOG.isDebugEnabled()) {
LOG.debug("Routing recovered event"
+ ", vertex=" + logIdentifier
+ ", eventType=" + tezEvent.getEventType()
+ ", sourceInfo=" + sourceMeta
- + ", destinationVertex" + destVertex.getName());
+ + ", destinationVertex=" + destVertex.getLogIdentifier());
}
eventHandler.handle(new VertexEventRouteEvent(destVertex
.getVertexId(), Collections.singletonList(tezEvent), true));
@@ -2495,8 +2495,8 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
Preconditions.checkState(
(vertex.sourceVertices == null || vertex.sourceVertices.containsKey(otherVertex) ||
vertex.targetVertices == null || vertex.targetVertices.containsKey(otherVertex)),
- "Not connected to vertex " + otherVertex.getName() + " from vertex: " + vertex.logIdentifier);
- LOG.info("Edge initialized for connection to vertex " + otherVertex.getName() +
+ "Not connected to vertex " + otherVertex.getLogIdentifier() + " from vertex: " + vertex.logIdentifier);
+ LOG.info("Edge initialized for connection to vertex " + otherVertex.getLogIdentifier() +
" at vertex : " + vertex.logIdentifier);
vertex.uninitializedEdges.remove(edge);
if(vertex.getState() == VertexState.INITIALIZING && vertex.canInitVertex()) {
@@ -2910,7 +2910,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
if (vertex.numTasks == -1) {
// this block must always return VertexState.INITIALIZING
LOG.info("Num tasks is -1. Expecting VertexManager/InputInitializers/1-1 split"
- + " to set #tasks for the vertex " + vertex.getVertexId());
+ + " to set #tasks for the vertex " + vertex.getLogIdentifier());
if (vertex.inputsWithInitializers != null) {
LOG.info("Vertex will initialize from input initializer. " + vertex.logIdentifier);
@@ -2935,12 +2935,12 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
LOG.info("Vertex will initialize via custom vertex manager. " + vertex.logIdentifier);
return VertexState.INITIALIZING;
}
- throw new TezUncheckedException(vertex.getVertexId() +
+ throw new TezUncheckedException(vertex.getLogIdentifier() +
" has -1 tasks but does not have input initializers, " +
"1-1 uninited sources or custom vertex manager to set it at runtime");
}
} else {
- LOG.info("Creating " + vertex.numTasks + " for vertex: " + vertex.logIdentifier);
+ LOG.info("Creating " + vertex.numTasks + " tasks for vertex: " + vertex.logIdentifier);
vertex.createTasks();
// this block may return VertexState.INITIALIZING
if (vertex.inputsWithInitializers != null) {
@@ -3082,7 +3082,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
.checkState(
(state == VertexState.INITIALIZING
|| state == VertexState.INITED || state == VertexState.RUNNING),
- " Unexpected 1-1 split for vertex " + vertex.getVertexId()
+ " Unexpected 1-1 split for vertex " + vertex.getLogIdentifier()
+ " in state " + vertex.getState() + " . Split in vertex "
+ originalSplitSource + " sent by vertex "
+ splitEvent.getSenderVertex() + " numTasks "
@@ -3090,19 +3090,19 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
if (vertex.originalOneToOneSplitSource.equals(originalSplitSource)) {
// ignore another split event that may have come from a different
// path in the DAG. We have already split because of that source
- LOG.info("Ignoring split of vertex " + vertex.getVertexId() +
+ LOG.info("Ignoring split of vertex " + vertex.getLogIdentifier() +
" because of split in vertex " + originalSplitSource +
" sent by vertex " + splitEvent.getSenderVertex() +
" numTasks " + splitEvent.getNumTasks());
return state;
}
// cannot split from multiple sources
- throw new TezUncheckedException("Vertex: " + vertex.getVertexId() +
+ throw new TezUncheckedException("Vertex: " + vertex.getLogIdentifier() +
" asked to split by: " + originalSplitSource +
" but was already split by:" + vertex.originalOneToOneSplitSource);
}
- LOG.info("Splitting vertex " + vertex.getVertexId() +
+ LOG.info("Splitting vertex " + vertex.getLogIdentifier() +
" because of split in vertex " + originalSplitSource +
" sent by vertex " + splitEvent.getSenderVertex() +
" numTasks " + splitEvent.getNumTasks());
@@ -3119,7 +3119,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
return vertex.getState();
} else {
Preconditions.checkState(vertex.getState() == VertexState.INITIALIZING,
- " Unexpected 1-1 split for vertex " + vertex.getVertexId() +
+ " Unexpected 1-1 split for vertex " + vertex.getLogIdentifier() +
" in state " + vertex.getState() +
" . Split in vertex " + originalSplitSource +
" sent by vertex " + splitEvent.getSenderVertex() +
@@ -3144,7 +3144,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
}
vertex.numStartedSourceVertices++;
LOG.info("Source vertex started: " + startEvent.getSourceVertexId() +
- " for vertex: " + vertex.getVertexId() + " numStartedSources: " +
+ " for vertex: " + vertex.logIdentifier + " numStartedSources: " +
vertex.numStartedSourceVertices + " numSources: " + vertex.sourceVertices.size());
if (vertex.numStartedSourceVertices < vertex.sourceVertices.size()) {
@@ -3463,7 +3463,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
public VertexState transition(VertexImpl vertex, VertexEvent event) {
VertexEventTaskAttemptCompleted completionEvent =
((VertexEventSourceTaskAttemptCompleted) event).getCompletionEvent();
- LOG.info("Source task attempt completed for vertex: " + vertex.getVertexId()
+ LOG.info("Source task attempt completed for vertex: " + vertex.getLogIdentifier()
+ " attempt: " + completionEvent.getTaskAttemptId()
+ " with state: " + completionEvent.getTaskAttemptState()
+ " vertexState: " + vertex.getState());
@@ -3620,7 +3620,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
if (vertex.outputCommitters == null // no committer
|| vertex.outputCommitters.isEmpty() // no committer
|| !vertex.commitVertexOutputs) { // committer does not commit on vertex success
- LOG.info(vertex.getVertexId() + " back to running due to rescheduling "
+ LOG.info(vertex.getLogIdentifier() + " back to running due to rescheduling "
+ ((VertexEventTaskReschedule)event).getTaskID());
(new TaskRescheduledTransition()).transition(vertex, event);
// inform the DAG that we are re-running
@@ -3629,7 +3629,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
}
// terminate any running tasks
- String diagnosticMsg = vertex.getVertexId() + " failed due to post-commit rescheduling of "
+ String diagnosticMsg = vertex.getLogIdentifier() + " failed due to post-commit rescheduling of "
+ ((VertexEventTaskReschedule)event).getTaskID();
LOG.info(diagnosticMsg);
vertex.tryEnactKill(VertexTerminationCause.OWN_TASK_FAILURE,
@@ -3747,7 +3747,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
if (destEdge == null) {
throw new TezUncheckedException("Bad destination vertex: " +
sourceMeta.getEdgeVertexName() + " for event vertex: " +
- vertex.getVertexId());
+ vertex.getLogIdentifier());
}
vertex.eventHandler.handle(new VertexEventRouteEvent(destVertex
.getVertexId(), Collections.singletonList(tezEvent)));
@@ -3762,7 +3762,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
if (srcEdge == null) {
throw new TezUncheckedException("Bad source vertex: " +
sourceMeta.getTaskVertexName() + " for destination vertex: " +
- vertex.getVertexId());
+ vertex.getLogIdentifier());
}
srcEdge.sendTezEventToDestinationTasks(tezEvent);
}
@@ -3876,10 +3876,10 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
@Override
public void transition(VertexImpl vertex, VertexEvent event) {
LOG.error("Invalid event " + event.getType() + " on Vertex "
- + vertex.getVertexId());
+ + vertex.getLogIdentifier());
vertex.eventHandler.handle(new DAGEventDiagnosticsUpdate(
vertex.getDAGId(), "Invalid event " + event.getType()
- + " on Vertex " + vertex.getVertexId()));
+ + " on Vertex " + vertex.getLogIdentifier()));
vertex.setFinishTime();
vertex.finished(VertexState.ERROR);
}
@@ -4118,7 +4118,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
for (Entry<Vertex, Edge> entry : this.getInputVertices().entrySet()) {
InputSpec inputSpec = entry.getValue().getDestinationSpec(taskIndex);
if (LOG.isDebugEnabled()) {
- LOG.debug("For vertex : " + this.getName()
+ LOG.debug("For vertex : " + this.getLogIdentifier()
+ ", Using InputSpec : " + inputSpec);
}
// TODO DAGAM This should be based on the edge type.
http://git-wip-us.apache.org/repos/asf/tez/blob/103839b6/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java
index dd38c2a..f02b73b 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java
@@ -179,7 +179,7 @@ public class VertexManager {
});
if (LOG.isDebugEnabled()) {
- LOG.debug("vertex:" + managedVertex.getName() + "; Added " + events.size() + " for input " +
+ LOG.debug("vertex:" + managedVertex.getLogIdentifier() + "; Added " + events.size() + " for input " +
"name " + inputName);
}
rootInputInitEventQueue.addAll(tezEvents);
@@ -402,7 +402,7 @@ public class VertexManager {
throw new AMUserCodeException(Source.VertexManager, e);
}
if (LOG.isDebugEnabled()) {
- LOG.debug("vertex:" + managedVertex.getName() + "; after call of VertexManagerPlugin.onRootVertexInitialized"
+ LOG.debug("vertex:" + managedVertex.getLogIdentifier() + "; after call of VertexManagerPlugin.onRootVertexInitialized"
+ " on input:" + inputName + ", current task events size is " + rootInputInitEventQueue.size());
}
List<TezEvent> resultEvents = new ArrayList<TezEvent>();
[40/50] [abbrv] tez git commit: Revert "TEZ-1642. TestAMRecovery
sometimes fail. (zjffdu)"
Posted by je...@apache.org.
Revert "TEZ-1642. TestAMRecovery sometimes fail. (zjffdu)"
This reverts commit 6dda1d77bcd6460780144ae2aab905c2a255e94a.
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/0af02131
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/0af02131
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/0af02131
Branch: refs/heads/TEZ-8
Commit: 0af021318a08ee60c9268ae94763776ccab87c8a
Parents: 8f8bac8
Author: Jeff Zhang <zj...@apache.org>
Authored: Tue Nov 11 10:42:03 2014 +0800
Committer: Jeff Zhang <zj...@apache.org>
Committed: Tue Nov 11 10:42:03 2014 +0800
----------------------------------------------------------------------
CHANGES.txt | 1 -
.../org/apache/tez/test/TestAMRecovery.java | 62 ++++++--------------
2 files changed, 19 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/0af02131/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 57ca0a4..57a0c7f 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -21,7 +21,6 @@ ALL CHANGES:
TEZ-1749. Increase test timeout for TestLocalMode.testMultipleClientsWithSession
TEZ-1750. Add a DAGScheduler which schedules tasks only when sources have been scheduled.
TEZ-1761. TestRecoveryParser::testGetLastInProgressDAG fails in similar manner to TEZ-1686.
- TEZ-1642. TestAMRecovery sometimes fail.
Release 0.5.2: 2014-11-07
http://git-wip-us.apache.org/repos/asf/tez/blob/0af02131/tez-tests/src/test/java/org/apache/tez/test/TestAMRecovery.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestAMRecovery.java b/tez-tests/src/test/java/org/apache/tez/test/TestAMRecovery.java
index 867c7ab..42d6f5c 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/TestAMRecovery.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/TestAMRecovery.java
@@ -104,7 +104,7 @@ public class TestAMRecovery {
}
if (miniTezCluster == null) {
miniTezCluster =
- new MiniTezCluster(TestAMRecovery.class.getName(), 2, 1, 1);
+ new MiniTezCluster(TestAMRecovery.class.getName(), 1, 1, 1);
Configuration miniTezconf = new Configuration(conf);
miniTezconf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, MAX_AM_ATTEMPT);
miniTezconf.set("fs.defaultFS", remoteFs.getUri().toString()); // use HDFS
@@ -127,7 +127,6 @@ public class TestAMRecovery {
try {
LOG.info("Stopping MiniTezCluster");
miniTezCluster.stop();
- miniTezCluster = null;
} catch (Exception e) {
e.printStackTrace();
}
@@ -166,7 +165,6 @@ public class TestAMRecovery {
tezConf.setBoolean(
RecoveryService.TEZ_AM_RECOVERY_HANDLE_REMAINING_EVENT_WHEN_STOPPED,
true);
- tezConf.setInt(TezConfiguration.TEZ_AM_RM_HEARTBEAT_INTERVAL_MS_MAX, 100);
tezSession = TezClient.create("TestDAGRecovery", tezConf);
tezSession.start();
}
@@ -184,16 +182,6 @@ public class TestAMRecovery {
tezSession = null;
}
- private void printHistoryEvents(List<HistoryEvent> historyEvents, int attemptId) {
- LOG.info("RecoveryLogs from attempt:" + attemptId);
- for(HistoryEvent historyEvent : historyEvents) {
- LOG.info("Parsed event from recovery stream"
- + ", eventType=" + historyEvent.getEventType()
- + ", event=" + historyEvent);
- }
- LOG.info("");
- }
-
/**
* Fine-grained recovery task-level, In a vertex (v1), task 0 is done task 1
* is running. History flush happens. AM dies. Once AM is recovered, task 0 is
@@ -207,16 +195,14 @@ public class TestAMRecovery {
createDAG(ControlledInputReadyVertexManager.class,
DataMovementType.BROADCAST, true);
TezCounters counters = runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
- List<HistoryEvent> historyEvents1 = readRecoveryLog(1);
- List<HistoryEvent> historyEvents2 = readRecoveryLog(2);
- printHistoryEvents(historyEvents1, 1);
- printHistoryEvents(historyEvents2, 2);
-
assertEquals(5, counters.findCounter(DAGCounter.TOTAL_LAUNCHED_TASKS).getValue());
assertEquals(1, counters.findCounter(DAGCounter.NUM_KILLED_TASKS).getValue());
assertEquals(4, counters.findCounter(DAGCounter.NUM_SUCCEEDED_TASKS).getValue());
assertEquals(2, counters.findCounter(TestCounter.Counter_1).getValue());
+ List<HistoryEvent> historyEvents1 = readRecoveryLog(1);
+ List<HistoryEvent> historyEvents2 = readRecoveryLog(2);
+
// task_0 of v1 is finished in attempt 1, task_1 of v1 is not finished in
// attempt 1
assertEquals(1, findTaskAttemptFinishedEvent(historyEvents1, 0, 0).size());
@@ -241,16 +227,14 @@ public class TestAMRecovery {
createDAG(ControlledInputReadyVertexManager.class,
DataMovementType.BROADCAST, false);
TezCounters counters = runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
- List<HistoryEvent> historyEvents1 = readRecoveryLog(1);
- List<HistoryEvent> historyEvents2 = readRecoveryLog(2);
- printHistoryEvents(historyEvents1, 1);
- printHistoryEvents(historyEvents2, 2);
-
assertEquals(4, counters.findCounter(DAGCounter.TOTAL_LAUNCHED_TASKS).getValue());
assertEquals(0, counters.findCounter(DAGCounter.NUM_KILLED_TASKS).getValue());
assertEquals(4, counters.findCounter(DAGCounter.NUM_SUCCEEDED_TASKS).getValue());
assertEquals(2, counters.findCounter(TestCounter.Counter_1).getValue());
+ List<HistoryEvent> historyEvents1 = readRecoveryLog(1);
+ List<HistoryEvent> historyEvents2 = readRecoveryLog(2);
+
// task_0 of v1 is finished in attempt 1, task_1 of v1 is not finished in
// attempt 1
assertEquals(1, findTaskAttemptFinishedEvent(historyEvents1, 0, 0).size());
@@ -275,16 +259,14 @@ public class TestAMRecovery {
createDAG(ControlledInputReadyVertexManager.class,
DataMovementType.ONE_TO_ONE, true);
TezCounters counters = runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
- List<HistoryEvent> historyEvents1 = readRecoveryLog(1);
- List<HistoryEvent> historyEvents2 = readRecoveryLog(2);
- printHistoryEvents(historyEvents1, 1);
- printHistoryEvents(historyEvents2, 2);
-
assertEquals(5, counters.findCounter(DAGCounter.TOTAL_LAUNCHED_TASKS).getValue());
assertEquals(1, counters.findCounter(DAGCounter.NUM_KILLED_TASKS).getValue());
assertEquals(4, counters.findCounter(DAGCounter.NUM_SUCCEEDED_TASKS).getValue());
assertEquals(2, counters.findCounter(TestCounter.Counter_1).getValue());
+ List<HistoryEvent> historyEvents1 = readRecoveryLog(1);
+ List<HistoryEvent> historyEvents2 = readRecoveryLog(2);
+
// task_0 of v1 is finished in attempt 1, task_1 of v1 is not finished in
// attempt 1
assertEquals(1, findTaskAttemptFinishedEvent(historyEvents1, 0, 0).size());
@@ -310,16 +292,14 @@ public class TestAMRecovery {
createDAG(ControlledInputReadyVertexManager.class,
DataMovementType.ONE_TO_ONE, false);
TezCounters counters = runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
- List<HistoryEvent> historyEvents1 = readRecoveryLog(1);
- List<HistoryEvent> historyEvents2 = readRecoveryLog(2);
- printHistoryEvents(historyEvents1, 1);
- printHistoryEvents(historyEvents2, 2);
-
assertEquals(4, counters.findCounter(DAGCounter.TOTAL_LAUNCHED_TASKS).getValue());
assertEquals(0, counters.findCounter(DAGCounter.NUM_KILLED_TASKS).getValue());
assertEquals(4, counters.findCounter(DAGCounter.NUM_SUCCEEDED_TASKS).getValue());
assertEquals(2, counters.findCounter(TestCounter.Counter_1).getValue());
+ List<HistoryEvent> historyEvents1 = readRecoveryLog(1);
+ List<HistoryEvent> historyEvents2 = readRecoveryLog(2);
+
// task_0 of v1 is finished in attempt 1, task_1 of v1 is not finished in
// attempt 1
assertEquals(1, findTaskAttemptFinishedEvent(historyEvents1, 0, 0).size());
@@ -345,16 +325,14 @@ public class TestAMRecovery {
createDAG(ControlledShuffleVertexManager.class,
DataMovementType.SCATTER_GATHER, true);
TezCounters counters = runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
- List<HistoryEvent> historyEvents1 = readRecoveryLog(1);
- List<HistoryEvent> historyEvents2 = readRecoveryLog(2);
- printHistoryEvents(historyEvents1, 1);
- printHistoryEvents(historyEvents2, 2);
-
assertEquals(5, counters.findCounter(DAGCounter.TOTAL_LAUNCHED_TASKS).getValue());
assertEquals(1, counters.findCounter(DAGCounter.NUM_KILLED_TASKS).getValue());
assertEquals(4, counters.findCounter(DAGCounter.NUM_SUCCEEDED_TASKS).getValue());
assertEquals(2, counters.findCounter(TestCounter.Counter_1).getValue());
+ List<HistoryEvent> historyEvents1 = readRecoveryLog(1);
+ List<HistoryEvent> historyEvents2 = readRecoveryLog(2);
+
// task_0 of v1 is finished in attempt 1, task_1 of v1 is not finished in
// attempt 1
assertEquals(1, findTaskAttemptFinishedEvent(historyEvents1, 0, 0).size());
@@ -380,16 +358,14 @@ public class TestAMRecovery {
createDAG(ControlledShuffleVertexManager.class,
DataMovementType.SCATTER_GATHER, false);
TezCounters counters = runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
- List<HistoryEvent> historyEvents1 = readRecoveryLog(1);
- List<HistoryEvent> historyEvents2 = readRecoveryLog(2);
- printHistoryEvents(historyEvents1, 1);
- printHistoryEvents(historyEvents2, 2);
-
assertEquals(4, counters.findCounter(DAGCounter.TOTAL_LAUNCHED_TASKS).getValue());
assertEquals(0, counters.findCounter(DAGCounter.NUM_KILLED_TASKS).getValue());
assertEquals(4, counters.findCounter(DAGCounter.NUM_SUCCEEDED_TASKS).getValue());
assertEquals(2, counters.findCounter(TestCounter.Counter_1).getValue());
+ List<HistoryEvent> historyEvents1 = readRecoveryLog(1);
+ List<HistoryEvent> historyEvents2 = readRecoveryLog(2);
+
// task_0 of v1 is finished in attempt 1, task_1 of v1 is not finished in
// attempt 1
assertEquals(1, findTaskAttemptFinishedEvent(historyEvents1, 0, 0).size());
[34/50] [abbrv] tez git commit: TEZ-1742 addendum patch for follow up
comments
Posted by je...@apache.org.
TEZ-1742 addendum patch for follow up comments
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/c6c08c18
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/c6c08c18
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/c6c08c18
Branch: refs/heads/TEZ-8
Commit: c6c08c18b77870e71298adb2dc51908473434768
Parents: 0127a90
Author: Bikas Saha <bi...@apache.org>
Authored: Sat Nov 8 11:55:05 2014 -0800
Committer: Bikas Saha <bi...@apache.org>
Committed: Sat Nov 8 11:55:22 2014 -0800
----------------------------------------------------------------------
.../dag/app/rm/YarnTaskSchedulerService.java | 16 +++++++++------
.../tez/dag/app/rm/TestTaskScheduler.java | 21 +++++++++++++++++++-
2 files changed, 30 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/c6c08c18/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java
index 5941a45..f8fbd53 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java
@@ -1051,8 +1051,8 @@ public class YarnTaskSchedulerService extends TaskSchedulerService
return false;
}
- private int scaleDownByPreemptionPercentage(int original) {
- return (original + (preemptionPercentage - 1)) / preemptionPercentage;
+ static int scaleDownByPreemptionPercentage(int original, int percent) {
+ return (int) Math.ceil((original * percent)/100.f);
}
void preemptIfNeeded() {
@@ -1103,8 +1103,9 @@ public class YarnTaskSchedulerService extends TaskSchedulerService
// free up some more
// TODO this is subject to error wrt RM resource normalization
- numPendingRequestsToService = scaleDownByPreemptionPercentage(numHighestPriRequests);
-
+ numPendingRequestsToService = scaleDownByPreemptionPercentage(numHighestPriRequests,
+ preemptionPercentage);
+
if (numPendingRequestsToService < 1) {
return;
}
@@ -1223,7 +1224,7 @@ public class YarnTaskSchedulerService extends TaskSchedulerService
}
if(preemptedTaskPriority != null) {
int newNumPendingRequestsToService = scaleDownByPreemptionPercentage(Math.min(
- numEntriesAtPreemptedPriority, numHighestPriRequests));
+ numEntriesAtPreemptedPriority, numHighestPriRequests), preemptionPercentage);
numPendingRequestsToService = Math.min(newNumPendingRequestsToService,
numPendingRequestsToService);
if (numPendingRequestsToService < 1) {
@@ -1237,8 +1238,11 @@ public class YarnTaskSchedulerService extends TaskSchedulerService
preemptedContainers = new ContainerId[numPendingRequestsToService];
int currIndex = 0;
for (Map.Entry<Object, Container> entry : taskAllocations.entrySet()) {
+ HeldContainer heldContainer = heldContainers.get(entry.getValue().getId());
+ CookieContainerRequest lastTaskInfo = heldContainer.getLastTaskInfo();
+ Priority taskPriority = lastTaskInfo.getPriority();
Container container = entry.getValue();
- if (preemptedTaskPriority.equals(container.getPriority())) {
+ if (preemptedTaskPriority.equals(taskPriority)) {
// taskAllocations map will iterate from oldest to newest assigned containers
// keep the N newest containersIds with the matching priority
preemptedContainers[currIndex++ % numPendingRequestsToService] = container.getId();
http://git-wip-us.apache.org/repos/asf/tez/blob/c6c08c18/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
index 3afff7c..5782d01 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
@@ -1328,7 +1328,7 @@ public class TestTaskScheduler {
Container mockContainer3B = mock(Container.class, RETURNS_DEEP_STUBS);
when(mockContainer3B.getNodeId().getHost()).thenReturn("host1");
when(mockContainer3B.getResource()).thenReturn(taskAsk);
- when(mockContainer3B.getPriority()).thenReturn(pri6);
+ when(mockContainer3B.getPriority()).thenReturn(pri2); // high priority container
ContainerId mockCId3B = mock(ContainerId.class);
when(mockContainer3B.getId()).thenReturn(mockCId3B);
containers.add(mockContainer3B);
@@ -1391,6 +1391,8 @@ public class TestTaskScheduler {
scheduler.taskAllocations.get(mockTask3).getId());
Assert.assertEquals(mockCId3A,
scheduler.taskAllocations.get(mockTask3KillA).getId());
+ // high priority container assigned to lower pri task. This task should still be preempted
+ // because the task priority is relevant for preemption and not the container priority
Assert.assertEquals(mockCId3B,
scheduler.taskAllocations.get(mockTask3KillB).getId());
@@ -1460,6 +1462,9 @@ public class TestTaskScheduler {
drainableAppCallback.drain();
// mockTaskPri3KillB gets preempted to clear 10% of outstanding running preemptable tasks
+ // this is also a higher priority container than the pending task priority but was running a
+ // lower priority task. Task priority is relevant for preemption and not container priority as
+ // containers can run tasks of different priorities
scheduler.getProgress();
drainableAppCallback.drain();
verify(mockRMClient, times(2)).releaseAssignedContainer((ContainerId)any());
@@ -1633,6 +1638,20 @@ public class TestTaskScheduler {
when(appClient.getFinalAppStatus()).thenReturn(finalStatus);
taskScheduler.close();
}
+
+ @Test (timeout=5000)
+ public void testScaleDownPercentage() {
+ Assert.assertEquals(100, YarnTaskSchedulerService.scaleDownByPreemptionPercentage(100, 100));
+ Assert.assertEquals(70, YarnTaskSchedulerService.scaleDownByPreemptionPercentage(100, 70));
+ Assert.assertEquals(50, YarnTaskSchedulerService.scaleDownByPreemptionPercentage(100, 50));
+ Assert.assertEquals(10, YarnTaskSchedulerService.scaleDownByPreemptionPercentage(100, 10));
+ Assert.assertEquals(5, YarnTaskSchedulerService.scaleDownByPreemptionPercentage(100, 5));
+ Assert.assertEquals(1, YarnTaskSchedulerService.scaleDownByPreemptionPercentage(100, 1));
+ Assert.assertEquals(1, YarnTaskSchedulerService.scaleDownByPreemptionPercentage(5, 5));
+ Assert.assertEquals(1, YarnTaskSchedulerService.scaleDownByPreemptionPercentage(1, 10));
+ Assert.assertEquals(1, YarnTaskSchedulerService.scaleDownByPreemptionPercentage(1, 70));
+ Assert.assertEquals(1, YarnTaskSchedulerService.scaleDownByPreemptionPercentage(1, 1));
+ }
private Container createContainer(int id, String host, Resource resource,
Priority priority) {
[24/50] [abbrv] tez git commit: TEZ-1738. Tez tfile parser for log
parsing (Rajesh Balamohan)
Posted by je...@apache.org.
TEZ-1738. Tez tfile parser for log parsing (Rajesh Balamohan)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/7b55e380
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/7b55e380
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/7b55e380
Branch: refs/heads/TEZ-8
Commit: 7b55e380895203406c01896b2c577fa12e621390
Parents: f4d50f8
Author: Rajesh Balamohan <rb...@apache.org>
Authored: Thu Nov 6 05:47:53 2014 -0800
Committer: Rajesh Balamohan <rb...@apache.org>
Committed: Thu Nov 6 05:47:53 2014 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
tez-tools/tez-tfile-parser/README.txt | 24 +++
tez-tools/tez-tfile-parser/pom.xml | 150 +++++++++++++++
.../java/org/apache/tez/tools/TFileLoader.java | 184 +++++++++++++++++++
.../org/apache/tez/tools/TFileRecordReader.java | 107 +++++++++++
5 files changed, 466 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/7b55e380/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 1b0a2d9..adc1a85 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
ALL CHANGES:
TEZ-1733. TezMerger should sort FileChunks on size when merging
+ TEZ-1738. Tez tfile parser for log parsing
Release 0.5.2: Unreleased
http://git-wip-us.apache.org/repos/asf/tez/blob/7b55e380/tez-tools/tez-tfile-parser/README.txt
----------------------------------------------------------------------
diff --git a/tez-tools/tez-tfile-parser/README.txt b/tez-tools/tez-tfile-parser/README.txt
new file mode 100644
index 0000000..09514ed
--- /dev/null
+++ b/tez-tools/tez-tfile-parser/README.txt
@@ -0,0 +1,24 @@
+It can be time consuming to download logs via "yarn logs -applicationId <appId> | grep something". Also mining large volumes of logs can be time consuming on single node.
+This is a simple Pig loader to parse TFiles and provide line by line format (tuple of (machine, key, line)) for distributed processing of logs.
+
+Build/Install:
+==============
+1. "mvn clean package" should create "tfile-parser-1.0-SNAPSHOT.jar" would be created in ./target directory
+
+Running pig with tez:
+====================
+1. Install pig
+2. $PIG_HOME/bin/pig -x tez (to open grunt shell)
+
+Sample pig script:
+==================
+set pig.splitCombination false;
+set tez.grouping.min-size 52428800;
+set tez.grouping.max-size 52428800;
+
+register 'tfile-parser-1.0-SNAPSHOT.jar';
+raw = load '/app-logs/root/logs/application_1411511669099_0769/*' using org.apache.tez.tools.TFileLoader() as (machine:chararray, key:chararray, line:chararray);
+filterByLine = FILTER raw BY (key MATCHES '.*container_1411511669099_0769_01_000001.*')
+ AND (line MATCHES '.*Shuffle.*');
+dump filterByLine;
+
http://git-wip-us.apache.org/repos/asf/tez/blob/7b55e380/tez-tools/tez-tfile-parser/pom.xml
----------------------------------------------------------------------
diff --git a/tez-tools/tez-tfile-parser/pom.xml b/tez-tools/tez-tfile-parser/pom.xml
new file mode 100644
index 0000000..18f55d0
--- /dev/null
+++ b/tez-tools/tez-tfile-parser/pom.xml
@@ -0,0 +1,150 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>pig-tez-tfile-parser</groupId>
+ <artifactId>tfile-parser</artifactId>
+ <version>1.0-SNAPSHOT</version>
+
+ <properties>
+ <tez-version>0.6.0-SNAPSHOT</tez-version>
+ <hadoop-version>2.4.0</hadoop-version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ <version>1.1.1.3</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.pig</groupId>
+ <artifactId>pig</artifactId>
+ <version>0.13.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-api</artifactId>
+ <version>${tez-version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-common</artifactId>
+ <version>${tez-version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-runtime-library</artifactId>
+ <version>${tez-version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-mapreduce</artifactId>
+ <version>${tez-version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop-version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <version>${hadoop-version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop-version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <version>${hadoop-version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-client</artifactId>
+ <version>${hadoop-version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-nodemanager</artifactId>
+ <version>${hadoop-version}</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ <version>1.1</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <version>1.8.5</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.7</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <repositories>
+ <repository>
+ <id>apache.snapshots</id>
+ <name>Apache Snapshot Repository</name>
+ <url>http://repository.apache.org/snapshots</url>
+ <releases>
+ <enabled>false</enabled>
+ </releases>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ </repository>
+ </repositories>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-dependencies</id>
+ <phase>package</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${project.build.directory}/lib</outputDirectory>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/tez/blob/7b55e380/tez-tools/tez-tfile-parser/src/main/java/org/apache/tez/tools/TFileLoader.java
----------------------------------------------------------------------
diff --git a/tez-tools/tez-tfile-parser/src/main/java/org/apache/tez/tools/TFileLoader.java b/tez-tools/tez-tfile-parser/src/main/java/org/apache/tez/tools/TFileLoader.java
new file mode 100644
index 0000000..9abe578
--- /dev/null
+++ b/tez-tools/tez-tfile-parser/src/main/java/org/apache/tez/tools/TFileLoader.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.tools;
+
+import com.google.common.base.Objects;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.pig.Expression;
+import org.apache.pig.FileInputLoadFunc;
+import org.apache.pig.LoadMetadata;
+import org.apache.pig.ResourceSchema;
+import org.apache.pig.ResourceStatistics;
+import org.apache.pig.StoreFunc;
+import org.apache.pig.StoreFuncInterface;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigFileInputFormat;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.impl.util.Utils;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.regex.Pattern;
+
+/**
+ * Simple pig loader function (mainly loader) which reads TFile and emits every line as a tuple of
+ * (machine, key, line). This would be beneficial if huge amount of logs need to be mined.
+ */
+public class TFileLoader extends FileInputLoadFunc implements LoadMetadata {
+
+ private static final Log LOG = LogFactory.getLog(TFileLoader.class);
+
+ private TFileRecordReader recReader = null;
+
+ private BufferedReader bufReader;
+ private Text currentKey;
+ private final TupleFactory tupleFactory = TupleFactory.getInstance();
+
+ private final Pattern PATTERN = Pattern.compile(":");
+
+ /**
+ * We get one complete TFile per KV read.
+ * Add a BufferedReader so that we can scan a line at a time.
+ *
+ * @throws java.io.IOException
+ * @throws InterruptedException
+ */
+ //TODO: tasks can sometime throw OOM when single TFile is way too large. Adjust mem accordinly.
+ private void setupReader() throws IOException, InterruptedException {
+ if (recReader.nextKeyValue() && bufReader == null) {
+ currentKey = recReader.getCurrentKey();
+ Text val = recReader.getCurrentValue();
+ bufReader = new BufferedReader(new StringReader(val.toString()));
+ }
+ }
+
+ @Override
+ public Tuple getNext() throws IOException {
+ try {
+ String line = readLine();
+ if (line != null) {
+ //machine, key, line
+ Tuple tuple = tupleFactory.newTuple(3);
+ if (currentKey != null) {
+ String[] data = PATTERN.split(currentKey.toString());
+ if (data == null || data.length != 2) {
+ LOG.warn("unable to parse " + currentKey.toString());
+ return null;
+ }
+ tuple.set(0, data[0]);
+ tuple.set(1, data[1]);
+ } else {
+ tuple.set(0, "");
+ tuple.set(1, "");
+ }
+ tuple.set(2, line); //line
+ return tuple;
+ }
+ } catch (IOException e) {
+ return null;
+ } catch (InterruptedException e) {
+ return null;
+ }
+ return null;
+ }
+
+ private String readLine() throws IOException, InterruptedException {
+ String line = null;
+ if (bufReader == null) {
+ setupReader();
+ }
+ line = bufReader.readLine();
+ if (line == null) { //end of stream. Move to the next reader
+ bufReader = null;
+ setupReader();
+ if (bufReader != null) {
+ line = bufReader.readLine();
+ }
+ }
+ return line;
+ }
+
+ public static class TFileInputFormat extends
+ PigFileInputFormat<Text, Text> {
+ @Override
+ public RecordReader<Text, Text> createRecordReader(InputSplit split,
+ TaskAttemptContext context) throws IOException,
+ InterruptedException {
+ return new TFileRecordReader();
+ }
+
+ }
+
+ @Override
+ public InputFormat getInputFormat() {
+ return new TFileInputFormat();
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(this);
+ }
+
+ @Override
+ public void prepareToRead(RecordReader reader, PigSplit split) {
+ recReader = (TFileRecordReader) reader;
+ }
+
+ @Override
+ public void setLocation(String location, Job job) throws IOException {
+ FileInputFormat.setInputPaths(job, location);
+ }
+
+ @Override
+ public String[] getPartitionKeys(String location, Job job)
+ throws IOException {
+ return null;
+ }
+
+ @Override
+ public ResourceSchema getSchema(String location, Job job)
+ throws IOException {
+ return Utils.getSchema(this, location, true, job);
+ }
+
+ @Override
+ public ResourceStatistics getStatistics(String location, Job job)
+ throws IOException {
+ return null;
+ }
+
+ @Override
+ public void setPartitionFilter(Expression plan) throws IOException {
+ throw new UnsupportedOperationException("setPartitionFilter() not yet supported");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/7b55e380/tez-tools/tez-tfile-parser/src/main/java/org/apache/tez/tools/TFileRecordReader.java
----------------------------------------------------------------------
diff --git a/tez-tools/tez-tfile-parser/src/main/java/org/apache/tez/tools/TFileRecordReader.java b/tez-tools/tez-tfile-parser/src/main/java/org/apache/tez/tools/TFileRecordReader.java
new file mode 100644
index 0000000..16625bb
--- /dev/null
+++ b/tez-tools/tez-tfile-parser/src/main/java/org/apache/tez/tools/TFileRecordReader.java
@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.tools;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.file.tfile.TFile;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+
+import java.io.IOException;
+
+/**
+ * Simple record reader which reads the TFile and emits it as key, value pair
+ */
+public class TFileRecordReader extends RecordReader<Text, Text> {
+
+ private static final Log LOG = LogFactory.getLog(TFileRecordReader.class);
+
+ private long start, end;
+
+ private Path splitPath;
+ private FSDataInputStream fin;
+ private TFile.Reader reader;
+ private TFile.Reader.Scanner scanner;
+
+ private Text key = new Text();
+ private Text value = new Text();
+
+ private BytesWritable valueBytesWritable = new BytesWritable();
+ private BytesWritable keyBytesWritable = new BytesWritable();
+
+ @Override public void initialize(InputSplit split, TaskAttemptContext context)
+ throws IOException, InterruptedException {
+ FileSplit fileSplit = (FileSplit) split;
+ LOG.info("Initializing TFileRecordReader : " + fileSplit.getPath().toString());
+ start = fileSplit.getStart();
+ end = start + fileSplit.getLength();
+
+ FileSystem fs = fileSplit.getPath().getFileSystem(context.getConfiguration());
+ splitPath = fileSplit.getPath();
+ fin = fs.open(splitPath);
+ reader = new TFile.Reader(fin, fs.getFileStatus(splitPath).getLen(),
+ context.getConfiguration());
+ scanner = reader.createScannerByByteRange(start, fileSplit.getLength());
+ }
+
+ @Override public boolean nextKeyValue() throws IOException, InterruptedException {
+ valueBytesWritable.setSize(0);
+ if (!scanner.advance()) {
+ value = null;
+ return false;
+ }
+ TFile.Reader.Scanner.Entry entry = scanner.entry();
+ //populate key, value
+ entry.getKey(keyBytesWritable);
+ StringBuilder k = new StringBuilder();
+ //split path contains the machine name. Create the key as splitPath + realKey
+ k.append(splitPath.getName()).append(":").append(new String(keyBytesWritable.getBytes()));
+ key.set(k.toString());
+ entry.getValue(valueBytesWritable);
+ value.set(valueBytesWritable.getBytes());
+ return true;
+ }
+
+ @Override public Text getCurrentKey() throws IOException, InterruptedException {
+ return key;
+ }
+
+ @Override public Text getCurrentValue() throws IOException, InterruptedException {
+ return value;
+ }
+
+ @Override public float getProgress() throws IOException, InterruptedException {
+ return ((fin.getPos() - start) * 1.0f) / ((end - start) * 1.0f);
+ }
+
+ @Override public void close() throws IOException {
+ IOUtils.closeQuietly(scanner);
+ IOUtils.closeQuietly(reader);
+ IOUtils.closeQuietly(fin);
+ }
+}
[05/50] [abbrv] tez git commit: TEZ-1700. Replace containerId from
TaskLocationHint with [TaskIndex+Vertex] based affinity (bikas)
Posted by je...@apache.org.
TEZ-1700. Replace containerId from TaskLocationHint with [TaskIndex+Vertex] based affinity (bikas)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/74c7f874
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/74c7f874
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/74c7f874
Branch: refs/heads/TEZ-8
Commit: 74c7f874d558f933fbf9697c537d204eb56806c9
Parents: 06e9f88
Author: Bikas Saha <bi...@apache.org>
Authored: Wed Oct 29 18:46:50 2014 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Wed Oct 29 18:46:50 2014 -0700
----------------------------------------------------------------------
CHANGES.txt | 4 ++
.../main/java/org/apache/tez/dag/api/DAG.java | 7 +-
.../apache/tez/dag/api/TaskLocationHint.java | 71 +++++++++++++++-----
.../tez/dag/api/TestTaskLocationHint.java | 62 +++++++++++++++++
.../tez/dag/app/dag/impl/TaskAttemptImpl.java | 1 +
.../dag/app/rm/TaskSchedulerEventHandler.java | 45 ++++++++++---
.../app/rm/TestTaskSchedulerEventHandler.java | 62 ++++++++++++++++-
.../vertexmanager/InputReadyVertexManager.java | 11 ++-
.../TestInputReadyVertexManager.java | 45 +++++++------
.../examples/BroadcastAndOneToOneExample.java | 1 -
10 files changed, 250 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/74c7f874/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 7bc96ce..ca1da2e 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -16,6 +16,8 @@ INCOMPATIBLE CHANGES
context.getUserPayload can now return null, apps may need to add defensive code.
TEZ-1699. Vertex.setParallelism should throw an exception for invalid
invocations
+ TEZ-1700. Replace containerId from TaskLocationHint with [TaskIndex+Vertex]
+ based affinity
ALL CHANGES:
TEZ-1620. Wait for application finish before stopping MiniTezCluster
@@ -85,6 +87,8 @@ ALL CHANGES:
context.getUserPayload can now return null, apps may need to add defensive code.
TEZ-1699. Vertex.setParallelism should throw an exception for invalid
invocations
+ TEZ-1700. Replace containerId from TaskLocationHint with [TaskIndex+Vertex]
+ based affinity
Release 0.5.1: 2014-10-02
http://git-wip-us.apache.org/repos/asf/tez/blob/74c7f874/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
index 9b428f0..34c27bf 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
@@ -719,11 +719,12 @@ public class DAG {
if (vertexLocationHint.getTaskLocationHints() != null) {
for (TaskLocationHint hint : vertexLocationHint.getTaskLocationHints()) {
PlanTaskLocationHint.Builder taskLocationHintBuilder = PlanTaskLocationHint.newBuilder();
-
- if (hint.getAffinitizedContainer() != null) {
+ // we can allow this later on if needed
+ if (hint.getAffinitizedTask() != null) {
throw new TezUncheckedException(
- "Container affinity may not be specified via the DAG API");
+ "Task based affinity may not be specified via the DAG API");
}
+
if (hint.getHosts() != null) {
taskLocationHintBuilder.addAllHost(hint.getHosts());
}
http://git-wip-us.apache.org/repos/asf/tez/blob/74c7f874/tez-api/src/main/java/org/apache/tez/dag/api/TaskLocationHint.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TaskLocationHint.java b/tez-api/src/main/java/org/apache/tez/dag/api/TaskLocationHint.java
index 02f7bf9..926b8fd 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TaskLocationHint.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TaskLocationHint.java
@@ -23,7 +23,7 @@ import java.util.Set;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
-import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import com.google.common.base.Preconditions;
@@ -36,16 +36,40 @@ import com.google.common.base.Preconditions;
@Evolving
public class TaskLocationHint {
- // Container to which to affinitize
- ContainerId containerId;
+ @Unstable
+ /**
+ * Specifies location affinity to the given vertex and given task in that vertex
+ */
+ public class TaskBasedLocationAffinity {
+ private String vertexName;
+ private int taskIndex;
+ public TaskBasedLocationAffinity(String vertexName, int taskIndex) {
+ this.vertexName = vertexName;
+ this.taskIndex = taskIndex;
+ }
+ public String getVertexName() {
+ return vertexName;
+ }
+ public int getTaskIndex() {
+ return taskIndex;
+ }
+ @Override
+ public String toString() {
+ return "[Vertex: " + vertexName + ", TaskIndex: " + taskIndex + "]";
+ }
+ }
+
// Host names if any to be used
private Set<String> hosts;
// Rack names if any to be used
private Set<String> racks;
+
+ private TaskBasedLocationAffinity affinitizedTask;
- private TaskLocationHint(ContainerId containerId) {
- Preconditions.checkNotNull(containerId);
- this.containerId = containerId;
+ private TaskLocationHint(String vertexName, int taskIndex) {
+ Preconditions.checkNotNull(vertexName);
+ Preconditions.checkArgument(taskIndex >= 0);
+ this.affinitizedTask = new TaskBasedLocationAffinity(vertexName, taskIndex);
}
private TaskLocationHint(Set<String> hosts, Set<String> racks) {
@@ -62,11 +86,17 @@ public class TaskLocationHint {
}
/**
- * Provide a location hint using a container to which the task may be affinitized
- * Tez will try to run the task on that container or node local to it
+ * Provide a location hint that affinitizes to the given task in the given vertex. Tez will try
+ * to run in the same container as the given task or node local to it. Locality may degrade to
+ * rack local or further depending on cluster resource allocations.<br>
+ * This is expected to be used only during dynamic optimizations via {@link VertexManagerPlugin}s
+ * and not in while creating the dag using the DAG API.
+ * @param vertexName
+ * @param taskIndex
+ * @return
*/
- public static TaskLocationHint createTaskLocationHint(ContainerId containerId) {
- return new TaskLocationHint(containerId);
+ public static TaskLocationHint createTaskLocationHint(String vertexName, int taskIndex) {
+ return new TaskLocationHint(vertexName, taskIndex);
}
/**
@@ -77,8 +107,8 @@ public class TaskLocationHint {
return new TaskLocationHint(hosts, racks);
}
- public ContainerId getAffinitizedContainer() {
- return containerId;
+ public TaskBasedLocationAffinity getAffinitizedTask() {
+ return affinitizedTask;
}
public Set<String> getHosts() {
@@ -99,9 +129,9 @@ public class TaskLocationHint {
result = ( racks != null) ?
prime * result + racks.hashCode() :
result + prime;
- result = ( containerId != null) ?
- prime * result + containerId.hashCode() :
- result + prime;
+ if (affinitizedTask != null) {
+ result = prime * result + affinitizedTask.getVertexName().hashCode() + affinitizedTask.getTaskIndex();
+ }
return result;
}
@@ -131,11 +161,16 @@ public class TaskLocationHint {
} else if (other.racks != null) {
return false;
}
- if (containerId != null) {
- if (!containerId.equals(other.containerId)) {
+ if (affinitizedTask != null) {
+ if (other.affinitizedTask == null) {
+ return false;
+ }
+ if (affinitizedTask.getTaskIndex() != other.affinitizedTask.getTaskIndex()) {
+ return false;
+ } else if (!affinitizedTask.getVertexName().equals(other.affinitizedTask.getVertexName())) {
return false;
}
- } else if (other.containerId != null) {
+ } else if (other.affinitizedTask != null){
return false;
}
return true;
http://git-wip-us.apache.org/repos/asf/tez/blob/74c7f874/tez-api/src/test/java/org/apache/tez/dag/api/TestTaskLocationHint.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/TestTaskLocationHint.java b/tez-api/src/test/java/org/apache/tez/dag/api/TestTaskLocationHint.java
new file mode 100644
index 0000000..d52e194
--- /dev/null
+++ b/tez-api/src/test/java/org/apache/tez/dag/api/TestTaskLocationHint.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.collect.Sets;
+
+public class TestTaskLocationHint {
+
+ @Test (timeout = 5000)
+ public void testEquality() {
+ TaskLocationHint t1 = TaskLocationHint.createTaskLocationHint("v1", 0);
+ TaskLocationHint t2 = TaskLocationHint.createTaskLocationHint("v1", 0);
+ TaskLocationHint t3 = TaskLocationHint.createTaskLocationHint("v2", 0);
+ TaskLocationHint t4 = TaskLocationHint.createTaskLocationHint("v1", 1);
+ TaskLocationHint t5 = TaskLocationHint.createTaskLocationHint(null, null);
+ TaskLocationHint t6 = TaskLocationHint.createTaskLocationHint(null, null);
+ TaskLocationHint t7 = TaskLocationHint.createTaskLocationHint(
+ Sets.newHashSet(new String[] {"n1", "n2"}), Sets.newHashSet(new String[] {"r1", "r2"}));
+ TaskLocationHint t8 = TaskLocationHint.createTaskLocationHint(
+ Sets.newHashSet(new String[] {"n1", "n2"}), Sets.newHashSet(new String[] {"r1", "r2"}));
+ TaskLocationHint t9 = TaskLocationHint.createTaskLocationHint(
+ Sets.newHashSet(new String[] {"n1", "n2"}), Sets.newHashSet(new String[] {"r1"}));
+ TaskLocationHint t10 = TaskLocationHint.createTaskLocationHint(
+ Sets.newHashSet(new String[] {"n1"}), Sets.newHashSet(new String[] {"r1", "r2"}));
+
+ Assert.assertEquals(t1, t2);
+ Assert.assertEquals(t5, t6);
+ Assert.assertEquals(t7, t8);
+ Assert.assertEquals(t2, t1);
+ Assert.assertEquals(t6, t5);
+ Assert.assertEquals(t8, t7);
+ Assert.assertNotEquals(t1, t3);
+ Assert.assertNotEquals(t3, t1);
+ Assert.assertNotEquals(t1, t4);
+ Assert.assertNotEquals(t4, t1);
+ Assert.assertNotEquals(t1, t5);
+ Assert.assertNotEquals(t5, t1);
+ Assert.assertNotEquals(t8, t9);
+ Assert.assertNotEquals(t9, t8);
+ Assert.assertNotEquals(t9, t10);
+ Assert.assertNotEquals(t10, t9);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/74c7f874/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
index 43be5aa..45ca543 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
@@ -788,6 +788,7 @@ public class TaskAttemptImpl implements TaskAttempt,
this.launchTime = tEvent.getStartTime();
recoveryStartEventSeen = true;
recoveredState = TaskAttemptState.RUNNING;
+ this.containerId = tEvent.getContainerId();
sendEvent(createDAGCounterUpdateEventTALaunched(this));
return recoveredState;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/74c7f874/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
index df66fa0..ec8e73f 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
@@ -43,12 +43,14 @@ import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.api.TaskLocationHint;
+import org.apache.tez.dag.api.TaskLocationHint.TaskBasedLocationAffinity;
import org.apache.tez.dag.api.client.DAGClientServer;
import org.apache.tez.dag.api.oldrecords.TaskAttemptState;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.DAGAppMaster;
import org.apache.tez.dag.app.DAGAppMasterState;
import org.apache.tez.dag.app.dag.TaskAttempt;
+import org.apache.tez.dag.app.dag.Vertex;
import org.apache.tez.dag.app.dag.event.DAGAppMasterEvent;
import org.apache.tez.dag.app.dag.event.DAGAppMasterEventSchedulingServiceError;
import org.apache.tez.dag.app.dag.event.DAGAppMasterEventType;
@@ -68,6 +70,8 @@ import org.apache.tez.dag.app.rm.node.AMNodeEventStateChanged;
import org.apache.tez.dag.app.rm.node.AMNodeEventTaskAttemptEnded;
import org.apache.tez.dag.app.rm.node.AMNodeEventTaskAttemptSucceeded;
+import com.google.common.base.Preconditions;
+
public class TaskSchedulerEventHandler extends AbstractService
implements TaskSchedulerAppCallback,
@@ -263,14 +267,29 @@ public class TaskSchedulerEventHandler extends AbstractService
String hosts[] = null;
String racks[] = null;
if (locationHint != null) {
- if (locationHint.getAffinitizedContainer() != null) {
- taskScheduler.allocateTask(taskAttempt,
- event.getCapability(),
- locationHint.getAffinitizedContainer(),
- Priority.newInstance(event.getPriority()),
- event.getContainerContext(),
- event);
- return;
+ TaskBasedLocationAffinity taskAffinity = locationHint.getAffinitizedTask();
+ if (taskAffinity != null) {
+ Vertex vertex = appContext.getCurrentDAG().getVertex(taskAffinity.getVertexName());
+ Preconditions.checkNotNull(vertex, "Invalid vertex in task based affinity " + taskAffinity
+ + " for attempt: " + taskAttempt.getID());
+ int taskIndex = taskAffinity.getTaskIndex();
+ Preconditions.checkState(taskIndex >=0 && taskIndex < vertex.getTotalTasks(),
+ "Invalid taskIndex in task based affinity " + taskAffinity
+ + " for attempt: " + taskAttempt.getID());
+ TaskAttempt affinityAttempt = vertex.getTask(taskIndex).getSuccessfulAttempt();
+ if (affinityAttempt != null) {
+ Preconditions.checkNotNull(affinityAttempt.getAssignedContainerID(), affinityAttempt.getID());
+ taskScheduler.allocateTask(taskAttempt,
+ event.getCapability(),
+ affinityAttempt.getAssignedContainerID(),
+ Priority.newInstance(event.getPriority()),
+ event.getContainerContext(),
+ event);
+ return;
+ }
+ LOG.info("Attempt: " + taskAttempt.getID() + " has task based affinity to " + taskAffinity
+ + " but no locality information exists for it. Ignoring hint.");
+ // fall through with null hosts/racks
} else {
hosts = (locationHint.getHosts() != null) ? locationHint
.getHosts().toArray(
@@ -325,6 +344,9 @@ public class TaskSchedulerEventHandler extends AbstractService
while (!stopEventHandling && !Thread.currentThread().isInterrupted()) {
try {
+ if (TaskSchedulerEventHandler.this.eventQueue.peek() == null) {
+ notifyForTest();
+ }
event = TaskSchedulerEventHandler.this.eventQueue.take();
} catch (InterruptedException e) {
if(!stopEventHandling) {
@@ -341,13 +363,18 @@ public class TaskSchedulerEventHandler extends AbstractService
// Kill the AM.
sendEvent(new DAGAppMasterEvent(DAGAppMasterEventType.INTERNAL_ERROR));
return;
+ } finally {
+ notifyForTest();
}
}
}
};
this.eventHandlingThread.start();
}
-
+
+ protected void notifyForTest() {
+ }
+
@Override
public void serviceStop() {
synchronized(this) {
http://git-wip-us.apache.org/repos/asf/tez/blob/74c7f874/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerEventHandler.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerEventHandler.java
index edd9ebf..4ec1916 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerEventHandler.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerEventHandler.java
@@ -18,27 +18,38 @@
package org.apache.tez.dag.app.rm;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.Event;
import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.tez.dag.api.TaskLocationHint;
import org.apache.tez.dag.api.client.DAGClientServer;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.dag.impl.TaskAttemptImpl;
+import org.apache.tez.dag.app.dag.impl.TaskImpl;
+import org.apache.tez.dag.app.dag.impl.VertexImpl;
import org.apache.tez.dag.app.rm.container.AMContainer;
import org.apache.tez.dag.app.rm.container.AMContainerEventCompleted;
import org.apache.tez.dag.app.rm.container.AMContainerEventType;
import org.apache.tez.dag.app.rm.container.AMContainerMap;
import org.apache.tez.dag.app.rm.container.ContainerSignatureMatcher;
+import org.apache.tez.dag.records.TezTaskAttemptID;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -58,6 +69,8 @@ public class TestTaskSchedulerEventHandler {
class MockTaskSchedulerEventHandler extends TaskSchedulerEventHandler {
+ AtomicBoolean notify = new AtomicBoolean(false);
+
public MockTaskSchedulerEventHandler(AppContext appContext,
DAGClientServer clientService, EventHandler eventHandler,
ContainerSignatureMatcher containerSignatureMatcher) {
@@ -70,6 +83,14 @@ public class TestTaskSchedulerEventHandler {
return mockTaskScheduler;
}
+ @Override
+ protected void notifyForTest() {
+ synchronized (notify) {
+ notify.set(true);
+ notify.notifyAll();
+ }
+ }
+
}
AppContext mockAppContext;
@@ -82,7 +103,7 @@ public class TestTaskSchedulerEventHandler {
@Before
public void setup() {
- mockAppContext = mock(AppContext.class);
+ mockAppContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
mockClientService = mock(DAGClientServer.class);
mockEventHandler = new TestEventHandler();
mockSigMatcher = mock(ContainerSignatureMatcher.class);
@@ -95,6 +116,45 @@ public class TestTaskSchedulerEventHandler {
}
@Test (timeout = 5000)
+ public void testTaskBasedAffinity() throws Exception {
+ Configuration conf = new Configuration(false);
+ schedulerHandler.init(conf);
+ schedulerHandler.start();
+
+ TaskAttemptImpl mockTaskAttempt = mock(TaskAttemptImpl.class);
+ TezTaskAttemptID taId = mock(TezTaskAttemptID.class);
+ String affVertexName = "srcVertex";
+ int affTaskIndex = 1;
+ TaskLocationHint locHint = TaskLocationHint.createTaskLocationHint(affVertexName, affTaskIndex);
+ VertexImpl affVertex = mock(VertexImpl.class);
+ TaskImpl affTask = mock(TaskImpl.class);
+ TaskAttemptImpl affAttempt = mock(TaskAttemptImpl.class);
+ ContainerId affCId = mock(ContainerId.class);
+ when(affVertex.getTotalTasks()).thenReturn(2);
+ when(affVertex.getTask(affTaskIndex)).thenReturn(affTask);
+ when(affTask.getSuccessfulAttempt()).thenReturn(affAttempt);
+ when(affAttempt.getAssignedContainerID()).thenReturn(affCId);
+ when(mockAppContext.getCurrentDAG().getVertex(affVertexName)).thenReturn(affVertex);
+ Resource resource = Resource.newInstance(100, 1);
+ AMSchedulerEventTALaunchRequest event = new AMSchedulerEventTALaunchRequest
+ (taId, resource, null, mockTaskAttempt, locHint, 3, null);
+ schedulerHandler.notify.set(false);
+ schedulerHandler.handle(event);
+ synchronized (schedulerHandler.notify) {
+ while (!schedulerHandler.notify.get()) {
+ schedulerHandler.notify.wait();
+ }
+ }
+
+ // verify mockTaskAttempt affinitized to expected affCId
+ verify(mockTaskScheduler, times(1)).allocateTask(mockTaskAttempt, resource, affCId,
+ Priority.newInstance(3), null, event);
+
+ schedulerHandler.stop();
+ schedulerHandler.close();
+ }
+
+ @Test (timeout = 5000)
public void testContainerPreempted() throws IOException {
Configuration conf = new Configuration(false);
schedulerHandler.init(conf);
http://git-wip-us.apache.org/repos/asf/tez/blob/74c7f874/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/InputReadyVertexManager.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/InputReadyVertexManager.java b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/InputReadyVertexManager.java
index 6633933..8f30276 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/InputReadyVertexManager.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/InputReadyVertexManager.java
@@ -24,7 +24,6 @@ import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.tez.dag.api.EdgeProperty;
import org.apache.tez.dag.api.EdgeProperty.DataMovementType;
import org.apache.tez.dag.api.InputDescriptor;
@@ -47,7 +46,7 @@ public class InputReadyVertexManager extends VertexManagerPlugin {
Map<String, SourceVertexInfo> srcVertexInfo = Maps.newHashMap();
boolean taskIsStarted[];
int oneToOneSrcTasksDoneCount[];
- Container oneToOneLocationHints[];
+ TaskLocationHint oneToOneLocationHints[];
int numOneToOneEdges;
public InputReadyVertexManager(VertexManagerPluginContext context) {
@@ -115,7 +114,7 @@ public class InputReadyVertexManager extends VertexManagerPlugin {
"Managed task number must equal 1-1 source task number");
}
oneToOneSrcTasksDoneCount = new int[oneToOneSrcTaskCount];
- oneToOneLocationHints = new Container[oneToOneSrcTaskCount];
+ oneToOneLocationHints = new TaskLocationHint[oneToOneSrcTaskCount];
}
for (Map.Entry<String, List<Integer>> entry : completions.entrySet()) {
@@ -149,7 +148,7 @@ public class InputReadyVertexManager extends VertexManagerPlugin {
oneToOneSrcTasksDoneCount[taskId.intValue()]++;
// keep the latest container that completed as the location hint
// After there is standard data size info available then use it
- oneToOneLocationHints[taskId.intValue()] = getContext().getTaskContainer(vertex, taskId);
+ oneToOneLocationHints[taskId.intValue()] = TaskLocationHint.createTaskLocationHint(vertex, taskId);
}
}
@@ -192,11 +191,11 @@ public class InputReadyVertexManager extends VertexManagerPlugin {
taskIsStarted[i] = true;
TaskLocationHint locationHint = null;
if (oneToOneLocationHints[i] != null) {
- locationHint = TaskLocationHint.createTaskLocationHint(oneToOneLocationHints[i].getId());
+ locationHint = oneToOneLocationHints[i];
}
LOG.info("Starting task " + i + " for vertex: "
+ getContext().getVertexName() + " with location: "
- + ((locationHint != null) ? locationHint.getAffinitizedContainer() : "null"));
+ + ((locationHint != null) ? locationHint.getAffinitizedTask() : "null"));
tasksToStart.add(new TaskWithLocationHint(new Integer(i), locationHint));
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/74c7f874/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestInputReadyVertexManager.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestInputReadyVertexManager.java b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestInputReadyVertexManager.java
index f57bdb0..c6981ed 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestInputReadyVertexManager.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestInputReadyVertexManager.java
@@ -101,21 +101,12 @@ public class TestInputReadyVertexManager {
InputDescriptor.create("in"));
String mockManagedVertexId = "Vertex";
- Container mockContainer1 = mock(Container.class);
- ContainerId mockCId1 = mock(ContainerId.class);
- when(mockContainer1.getId()).thenReturn(mockCId1);
- Container mockContainer2 = mock(Container.class);
- ContainerId mockCId2 = mock(ContainerId.class);
- when(mockContainer2.getId()).thenReturn(mockCId2);
VertexManagerPluginContext mockContext = mock(VertexManagerPluginContext.class);
when(mockContext.getInputVertexEdgeProperties()).thenReturn(mockInputVertices);
when(mockContext.getVertexName()).thenReturn(mockManagedVertexId);
when(mockContext.getVertexNumTasks(mockManagedVertexId)).thenReturn(3);
when(mockContext.getVertexNumTasks(mockSrcVertexId1)).thenReturn(3);
- when(mockContext.getTaskContainer(mockSrcVertexId1, 0)).thenReturn(mockContainer1);
- when(mockContext.getTaskContainer(mockSrcVertexId1, 1)).thenReturn(mockContainer2);
- when(mockContext.getTaskContainer(mockSrcVertexId1, 2)).thenReturn(mockContainer1);
mockInputVertices.put(mockSrcVertexId1, eProp1);
Map<String, List<Integer>> initialCompletions = Maps.newHashMap();
@@ -127,20 +118,26 @@ public class TestInputReadyVertexManager {
verify(mockContext, times(1)).scheduleVertexTasks(requestCaptor.capture());
Assert.assertEquals(1, requestCaptor.getValue().size());
Assert.assertEquals(0, requestCaptor.getValue().get(0).getTaskIndex().intValue());
- Assert.assertEquals(mockCId1, requestCaptor.getValue().get(0)
- .getTaskLocationHint().getAffinitizedContainer());
+ Assert.assertEquals(mockSrcVertexId1, requestCaptor.getValue().get(0)
+ .getTaskLocationHint().getAffinitizedTask().getVertexName());
+ Assert.assertEquals(0, requestCaptor.getValue().get(0)
+ .getTaskLocationHint().getAffinitizedTask().getTaskIndex());
manager.onSourceTaskCompleted(mockSrcVertexId1, 1);
verify(mockContext, times(2)).scheduleVertexTasks(requestCaptor.capture());
Assert.assertEquals(1, requestCaptor.getValue().size());
Assert.assertEquals(1, requestCaptor.getValue().get(0).getTaskIndex().intValue());
- Assert.assertEquals(mockCId2, requestCaptor.getValue().get(0)
- .getTaskLocationHint().getAffinitizedContainer());
+ Assert.assertEquals(mockSrcVertexId1, requestCaptor.getValue().get(0)
+ .getTaskLocationHint().getAffinitizedTask().getVertexName());
+ Assert.assertEquals(1, requestCaptor.getValue().get(0)
+ .getTaskLocationHint().getAffinitizedTask().getTaskIndex());
manager.onSourceTaskCompleted(mockSrcVertexId1, 2);
verify(mockContext, times(3)).scheduleVertexTasks(requestCaptor.capture());
Assert.assertEquals(1, requestCaptor.getValue().size());
Assert.assertEquals(2, requestCaptor.getValue().get(0).getTaskIndex().intValue());
- Assert.assertEquals(mockCId1, requestCaptor.getValue().get(0)
- .getTaskLocationHint().getAffinitizedContainer());
+ Assert.assertEquals(mockSrcVertexId1, requestCaptor.getValue().get(0)
+ .getTaskLocationHint().getAffinitizedTask().getVertexName());
+ Assert.assertEquals(2, requestCaptor.getValue().get(0)
+ .getTaskLocationHint().getAffinitizedTask().getTaskIndex());
}
@Test (timeout=5000)
@@ -234,15 +231,19 @@ public class TestInputReadyVertexManager {
verify(mockContext, times(1)).scheduleVertexTasks(requestCaptor.capture());
Assert.assertEquals(1, requestCaptor.getValue().size());
Assert.assertEquals(0, requestCaptor.getValue().get(0).getTaskIndex().intValue());
- Assert.assertEquals(mockCId3, requestCaptor.getValue().get(0)
- .getTaskLocationHint().getAffinitizedContainer()); // affinity to last completion
+ Assert.assertEquals(mockSrcVertexId3, requestCaptor.getValue().get(0)
+ .getTaskLocationHint().getAffinitizedTask().getVertexName());
+ Assert.assertEquals(0, requestCaptor.getValue().get(0)
+ .getTaskLocationHint().getAffinitizedTask().getTaskIndex()); // affinity to last completion
// 1-1 completion triggers since other 1-1 is done
manager.onSourceTaskCompleted(mockSrcVertexId3, 1);
verify(mockContext, times(2)).scheduleVertexTasks(requestCaptor.capture());
Assert.assertEquals(1, requestCaptor.getValue().size());
Assert.assertEquals(1, requestCaptor.getValue().get(0).getTaskIndex().intValue());
- Assert.assertEquals(mockCId3, requestCaptor.getValue().get(0)
- .getTaskLocationHint().getAffinitizedContainer()); // affinity to last completion
+ Assert.assertEquals(mockSrcVertexId3, requestCaptor.getValue().get(0)
+ .getTaskLocationHint().getAffinitizedTask().getVertexName());
+ Assert.assertEquals(1, requestCaptor.getValue().get(0)
+ .getTaskLocationHint().getAffinitizedTask().getTaskIndex()); // affinity to last completion
// 1-1 completion does not trigger since other 1-1 is not done
manager.onSourceTaskCompleted(mockSrcVertexId3, 2);
verify(mockContext, times(2)).scheduleVertexTasks(anyList());
@@ -251,8 +252,10 @@ public class TestInputReadyVertexManager {
verify(mockContext, times(3)).scheduleVertexTasks(requestCaptor.capture());
Assert.assertEquals(1, requestCaptor.getValue().size());
Assert.assertEquals(2, requestCaptor.getValue().get(0).getTaskIndex().intValue());
- Assert.assertEquals(mockCId2, requestCaptor.getValue().get(0)
- .getTaskLocationHint().getAffinitizedContainer()); // affinity to last completion
+ Assert.assertEquals(mockSrcVertexId2, requestCaptor.getValue().get(0)
+ .getTaskLocationHint().getAffinitizedTask().getVertexName());
+ Assert.assertEquals(2, requestCaptor.getValue().get(0)
+ .getTaskLocationHint().getAffinitizedTask().getTaskIndex()); // affinity to last completion
// no more starts
manager.onSourceTaskCompleted(mockSrcVertexId3, 2);
http://git-wip-us.apache.org/repos/asf/tez/blob/74c7f874/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
index 42b31f1..69711d0 100644
--- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
@@ -50,7 +50,6 @@ import org.apache.tez.runtime.api.ObjectRegistry;
import org.apache.tez.runtime.api.ProcessorContext;
import org.apache.tez.runtime.library.api.KeyValueReader;
import org.apache.tez.runtime.library.api.KeyValueWriter;
-import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
import org.apache.tez.runtime.library.conf.UnorderedKVEdgeConfig;
import org.apache.tez.runtime.library.output.UnorderedKVOutput;
import org.apache.tez.runtime.library.processor.SimpleProcessor;