You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2014/02/18 23:47:55 UTC
git commit: TEZ-787. Revert Guava dependency to 11.0.2. (sseth)
Repository: incubator-tez
Updated Branches:
refs/heads/master 06bc6eabe -> 87f3ea351
TEZ-787. Revert Guava dependency to 11.0.2. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/87f3ea35
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/87f3ea35
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/87f3ea35
Branch: refs/heads/master
Commit: 87f3ea351db18cde8c50385dde37e69f534515ad
Parents: 06bc6ea
Author: Siddharth Seth <ss...@apache.org>
Authored: Tue Feb 18 14:47:24 2014 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Tue Feb 18 14:47:24 2014 -0800
----------------------------------------------------------------------
pom.xml | 2 +-
.../api/events/CompositeDataMovementEvent.java | 45 +++++++++++++-------
.../apache/tez/dag/app/rm/TaskScheduler.java | 5 ++-
.../common/resources/MemoryDistributor.java | 5 ++-
4 files changed, 38 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/87f3ea35/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 30d818a..83b39ad 100644
--- a/pom.xml
+++ b/pom.xml
@@ -297,7 +297,7 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <version>15.0</version>
+ <version>11.0.2</version>
</dependency>
<dependency>
<groupId>org.codehaus.jettison</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/87f3ea35/tez-api/src/main/java/org/apache/tez/runtime/api/events/CompositeDataMovementEvent.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/CompositeDataMovementEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/CompositeDataMovementEvent.java
index f7d7878..2abf1ff 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/CompositeDataMovementEvent.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/CompositeDataMovementEvent.java
@@ -18,13 +18,9 @@
package org.apache.tez.runtime.api.events;
-import org.apache.tez.runtime.api.Event;
+import java.util.Iterator;
-import com.google.common.collect.ContiguousSet;
-import com.google.common.collect.DiscreteDomain;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Range;
-import com.google.common.base.Function;
+import org.apache.tez.runtime.api.Event;
/**
* A convenience class to specify multiple DataMovementEvents which share the
@@ -83,16 +79,35 @@ public class CompositeDataMovementEvent extends Event {
}
public Iterable<DataMovementEvent> getEvents() {
- // Overkill. For now is enough to just run a loop over the ints.
- Range<Integer> range = Range.closedOpen(sourceIndexStart, sourceIndexEnd);
- ContiguousSet<Integer> intRange = ContiguousSet.create(range, DiscreteDomain.integers());
- return Iterables.transform(intRange, new Function<Integer, DataMovementEvent>() {
- public DataMovementEvent apply(Integer integer) {
- DataMovementEvent dmEvent = new DataMovementEvent(integer, userPayload);
- dmEvent.setVersion(version);
- return dmEvent;
+
+ return new Iterable<DataMovementEvent>() {
+
+ @Override
+ public Iterator<DataMovementEvent> iterator() {
+ return new Iterator<DataMovementEvent>() {
+
+ int currentPos = sourceIndexStart;
+
+ @Override
+ public boolean hasNext() {
+ return currentPos < sourceIndexEnd;
+ }
+
+ @Override
+ public DataMovementEvent next() {
+ DataMovementEvent dmEvent = new DataMovementEvent(currentPos, userPayload);
+ currentPos++;
+ dmEvent.setVersion(version);
+ return dmEvent;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException("Remove not supported");
+ }
+ };
}
- });
+ };
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/87f3ea35/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
index 52173f6..814af6b 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
@@ -21,6 +21,7 @@ package org.apache.tez.dag.app.rm;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Collection;
+import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
@@ -29,6 +30,7 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.PriorityBlockingQueue;
@@ -147,7 +149,8 @@ public class TaskScheduler extends AbstractService
Map<ContainerId, HeldContainer> heldContainers =
new HashMap<ContainerId, HeldContainer>();
- Set<NodeId> blacklistedNodes = Sets.newConcurrentHashSet();
+ Set<NodeId> blacklistedNodes = Collections
+ .newSetFromMap(new ConcurrentHashMap<NodeId, Boolean>());
Resource totalResources = Resource.newInstance(0, 0);
Resource allocatedResources = Resource.newInstance(0, 0);
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/87f3ea35/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java
index 51d2de8..91a0b22 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java
@@ -24,6 +24,7 @@ import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.logging.Log;
@@ -44,7 +45,6 @@ import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
// Not calling this a MemoryManager explicitly. Not yet anyway.
@Private
@@ -63,7 +63,8 @@ public class MemoryDistributor {
private final boolean isEnabled;
private final boolean reserveFractionConfigured;
private float reserveFraction;
- private final Set<TezTaskContext> dupSet = Sets.newConcurrentHashSet();
+ private final Set<TezTaskContext> dupSet = Collections
+ .newSetFromMap(new ConcurrentHashMap<TezTaskContext, Boolean>());
private final List<RequestorInfo> requestList;
// Maybe make the reserve fraction configurable. Or scale it based on JVM heap.