You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by je...@apache.org on 2017/02/01 00:06:31 UTC
[1/2] tez git commit: TEZ-3597. Composite Fetch hangs on certain DME
empty events. (jeagles)
Repository: tez
Updated Branches:
refs/heads/TEZ-3334 3b20be06f -> bc3c17a57
TEZ-3597. Composite Fetch hangs on certain DME empty events. (jeagles)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/3a1f2c60
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/3a1f2c60
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/3a1f2c60
Branch: refs/heads/TEZ-3334
Commit: 3a1f2c60ffa0dcf462b84e9f6d5fd42d25c4d3c1
Parents: 3b20be0
Author: Jonathan Eagles <je...@yahoo-inc.com>
Authored: Tue Jan 31 18:03:42 2017 -0600
Committer: Jonathan Eagles <je...@yahoo-inc.com>
Committed: Tue Jan 31 18:03:42 2017 -0600
----------------------------------------------------------------------
TEZ-3334-CHANGES.txt | 1 +
.../shuffle/orderedgrouped/ShuffleScheduler.java | 18 +++++++++++++++---
2 files changed, 16 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/3a1f2c60/TEZ-3334-CHANGES.txt
----------------------------------------------------------------------
diff --git a/TEZ-3334-CHANGES.txt b/TEZ-3334-CHANGES.txt
index e40759f..8b55322 100644
--- a/TEZ-3334-CHANGES.txt
+++ b/TEZ-3334-CHANGES.txt
@@ -4,6 +4,7 @@ Apache Tez Change Log
INCOMPATIBLE CHANGES:
ALL CHANGES:
+ TEZ-3597. Composite Fetch hangs on certain DME empty events.
TEZ-3595. Composite Fetch account error for disk direct
TEZ-3590. Remove google.protobuf from the tez-auxservices shaded jar
TEZ-3587. Fetcher fetchInputs() can NPE on srcAttempt due to missing entry in pathToAttemptMap
http://git-wip-us.apache.org/repos/asf/tez/blob/3a1f2c60/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleScheduler.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleScheduler.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleScheduler.java
index 129e0cc..6e42bca 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleScheduler.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleScheduler.java
@@ -1110,8 +1110,14 @@ class ShuffleScheduler {
}
private boolean inputShouldBeConsumed(InputAttemptIdentifier id) {
- return (!obsoleteInputs.contains(id) &&
- !isInputFinished(id.getInputIdentifier()));
+ boolean isInputFinished = false;
+ if (id instanceof CompositeInputAttemptIdentifier) {
+ CompositeInputAttemptIdentifier cid = (CompositeInputAttemptIdentifier)id;
+ isInputFinished = isInputFinished(cid.getInputIdentifier(), cid.getInputIdentifier() + cid.getInputIdentifierCount());
+ } else {
+ isInputFinished = isInputFinished(id.getInputIdentifier());
+ }
+ return !obsoleteInputs.contains(id) && !isInputFinished;
}
public synchronized List<InputAttemptIdentifier> getMapsForHost(MapHost host) {
@@ -1286,13 +1292,19 @@ class ShuffleScheduler {
finishedMaps.set(inputIndex, true);
}
}
-
+
boolean isInputFinished(int inputIndex) {
synchronized (finishedMaps) {
return finishedMaps.get(inputIndex);
}
}
+ boolean isInputFinished(int inputIndex, int inputEnd) {
+ synchronized (finishedMaps) {
+ return finishedMaps.nextClearBit(inputIndex) >= inputEnd;
+ }
+ }
+
private class ShuffleSchedulerCallable extends CallableWithNdc<Void> {
[2/2] tez git commit: TEZ-3596. Number of Empty DME logged for
Composite fetch is too high (jeagles)
Posted by je...@apache.org.
TEZ-3596. Number of Empty DME logged for Composite fetch is too high (jeagles)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/bc3c17a5
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/bc3c17a5
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/bc3c17a5
Branch: refs/heads/TEZ-3334
Commit: bc3c17a57ec99697ce08a334c2341b3fcaacf386
Parents: 3a1f2c6
Author: Jonathan Eagles <je...@yahoo-inc.com>
Authored: Tue Jan 31 18:04:30 2017 -0600
Committer: Jonathan Eagles <je...@yahoo-inc.com>
Committed: Tue Jan 31 18:04:30 2017 -0600
----------------------------------------------------------------------
TEZ-3334-CHANGES.txt | 1 +
.../common/shuffle/impl/ShuffleInputEventHandlerImpl.java | 2 +-
.../orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java | 4 ++--
3 files changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/bc3c17a5/TEZ-3334-CHANGES.txt
----------------------------------------------------------------------
diff --git a/TEZ-3334-CHANGES.txt b/TEZ-3334-CHANGES.txt
index 8b55322..6c9a858 100644
--- a/TEZ-3334-CHANGES.txt
+++ b/TEZ-3334-CHANGES.txt
@@ -4,6 +4,7 @@ Apache Tez Change Log
INCOMPATIBLE CHANGES:
ALL CHANGES:
+ TEZ-3596. Number of Empty DME logged for Composite fetch is too high
TEZ-3597. Composite Fetch hangs on certain DME empty events.
TEZ-3595. Composite Fetch account error for disk direct
TEZ-3590. Remove google.protobuf from the tez-auxservices shaded jar
http://git-wip-us.apache.org/repos/asf/tez/blob/bc3c17a5/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java
index a80d21b..ee06de5 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java
@@ -180,7 +180,7 @@ public class ShuffleInputEventHandlerImpl implements ShuffleEventHandler {
LOG.debug("Source partition: " + srcIndex + " did not generate any data. SrcAttempt: ["
+ srcAttemptIdentifier + "]. Not fetching.");
}
- numDmeEventsNoData.incrementAndGet();
+ numDmeEventsNoData.getAndIncrement();
shuffleManager.addCompletedInputWithNoData(srcAttemptIdentifier.expand(0));
return;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/bc3c17a5/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java
index d34fb5f..cc24db9 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java
@@ -161,7 +161,7 @@ public class ShuffleInputEventHandlerOrderedGrouped implements ShuffleEventHandl
"Source partition: " + partitionId + " did not generate any data. SrcAttempt: ["
+ srcAttemptIdentifier + "]. Not fetching.");
}
- numDmeEventsNoData.incrementAndGet();
+ numDmeEventsNoData.getAndIncrement();
scheduler.copySucceeded(srcAttemptIdentifier.expand(0), null, 0, 0, 0, null, true);
return;
}
@@ -194,7 +194,7 @@ public class ShuffleInputEventHandlerOrderedGrouped implements ShuffleEventHandl
LOG.debug("Source partition: " + partitionId + " did not generate any data. SrcAttempt: ["
+ compositeInputAttemptIdentifier + "]. Not fetching.");
}
- numDmeEventsNoData.addAndGet(crdmEvent.getCount());
+ numDmeEventsNoData.getAndIncrement();
scheduler.copySucceeded(compositeInputAttemptIdentifier.expand(i), null, 0, 0, 0, null, true);
}
}