You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by jl...@apache.org on 2016/08/17 15:09:37 UTC
tez git commit: TEZ-3413. ConcurrentModificationException in
HistoryEventTimelineConversion for AppLaunchedEvent. (Jonathan Eagles via
jlowe)
Repository: tez
Updated Branches:
refs/heads/master d3fd82832 -> 1468457e6
TEZ-3413. ConcurrentModificationException in HistoryEventTimelineConversion for AppLaunchedEvent. (Jonathan Eagles via jlowe)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/1468457e
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/1468457e
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/1468457e
Branch: refs/heads/master
Commit: 1468457e671dfcf2780cf17d2655e086aef120c9
Parents: d3fd828
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Aug 17 15:08:34 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Wed Aug 17 15:08:34 2016 +0000
----------------------------------------------------------------------
CHANGES.txt | 3 ++
.../apache/tez/dag/history/utils/DAGUtils.java | 3 +-
.../ats/TestHistoryEventTimelineConversion.java | 34 ++++++++++++++++++++
3 files changed, 39 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/1468457e/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 62c69b0..a8e7080 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-3413. ConcurrentModificationException in HistoryEventTimelineConversion for AppLaunchedEvent.
TEZ-3352. MRInputHelpers getStringProperty() should not fail if property value is null.
TEZ-3409. Log dagId along with other information when submitting a dag.
TEZ-3384. Fix TestATSV15HistoryLoggingService::testDAGGroupingGroupingEnabled unit test.
@@ -97,6 +98,7 @@ INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-3413. ConcurrentModificationException in HistoryEventTimelineConversion for AppLaunchedEvent.
TEZ-3352. MRInputHelpers getStringProperty() should not fail if property value is null.
TEZ-3409. Log dagId along with other information when submitting a dag.
TEZ-3384. Fix TestATSV15HistoryLoggingService::testDAGGroupingGroupingEnabled unit test.
@@ -579,6 +581,7 @@ INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-3413. ConcurrentModificationException in HistoryEventTimelineConversion for AppLaunchedEvent.
TEZ-3286. Allow clients to set processor reserved memory per vertex (instead of per container).
TEZ-3223. Support a NullHistoryLogger to disable history logging if needed.
TEZ-3293. Fetch failures can cause a shuffle hang waiting for memory merge that never starts.
http://git-wip-us.apache.org/repos/asf/tez/blob/1468457e/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
index f1ac0ab..d8d2407 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
@@ -475,7 +475,8 @@ public class DAGUtils {
}
public static Map<String, String> convertConfigurationToATSMap(Configuration conf) {
- Iterator<Entry<String, String>> iter = conf.iterator();
+ // Copy configuration to avoid CME since iterator is not thread safe until HADOOP-13500
+ Iterator<Entry<String, String>> iter = new Configuration(conf).iterator();
Map<String, String> atsConf = new TreeMap<String, String>();
while (iter.hasNext()) {
Entry<String, String> entry = iter.next();
http://git-wip-us.apache.org/repos/asf/tez/blob/1468457e/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
index dabd6a5..62fb335 100644
--- a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
+++ b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java
@@ -23,6 +23,7 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -232,6 +233,39 @@ public class TestHistoryEventTimelineConversion {
}
+ @Test(timeout = 5000)
+ public void testConvertAppLaunchedEventConcurrentModificationException()
+ throws InterruptedException {
+ long launchTime = random.nextLong();
+ long submitTime = random.nextLong();
+ final Configuration conf = new Configuration(false);
+ conf.set("foo", "bar");
+ conf.set("applicationId", "1234");
+ final AtomicBoolean shutdown = new AtomicBoolean(false);
+
+ Thread confChanger = new Thread() {
+ @Override
+ public void run() {
+ int i = 1;
+ while (!shutdown.get()) {
+ // trigger an actual change to test concurrency with Configuration#iterator
+ conf.set("test" + i++, "test");
+ }
+ }
+ };
+
+ confChanger.start();
+ try {
+ MockVersionInfo mockVersionInfo = new MockVersionInfo();
+ AppLaunchedEvent event = new AppLaunchedEvent(applicationId, launchTime,
+ submitTime, user, conf, mockVersionInfo);
+ HistoryEventTimelineConversion.convertToTimelineEntity(event);
+ } finally {
+ shutdown.set(true);
+ confChanger.join();
+ }
+ }
+
@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testConvertAppLaunchedEvent() {