You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by je...@apache.org on 2021/02/17 21:47:10 UTC
[tez] branch master updated: TEZ-4289: Remove Dependency on
commons-math (#110)
This is an automated email from the ASF dual-hosted git repository.
jeagles pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tez.git
The following commit(s) were added to refs/heads/master by this push:
new 70623f0 TEZ-4289: Remove Dependency on commons-math (#110)
70623f0 is described below
commit 70623f0d8e9de3c59b61c10abe5442bfd8f66e2c
Author: belugabehr <12...@users.noreply.github.com>
AuthorDate: Wed Feb 17 16:47:00 2021 -0500
TEZ-4289: Remove Dependency on commons-math (#110)
* TEZ-4289: Remove Dependency on commons-math
* Remove stray reference to commons-math
* Fix checkstyle and missing parenthesis
---
pom.xml | 5 -----
tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java | 5 ++---
tez-dag/pom.xml | 4 ----
.../java/org/apache/tez/dag/app/rm/DagAwareYarnTaskScheduler.java | 5 ++---
.../java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java | 5 ++---
5 files changed, 6 insertions(+), 18 deletions(-)
diff --git a/pom.xml b/pom.xml
index 9fa2cb4..19fd516 100644
--- a/pom.xml
+++ b/pom.xml
@@ -669,11 +669,6 @@
<version>4.1</version>
</dependency>
<dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-math3</artifactId>
- <version>3.1.1</version>
- </dependency>
- <dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
<version>1.2</version>
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index 79069ed..389b584 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -43,7 +43,6 @@ import java.util.Objects;
import com.google.common.base.Strings;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.math3.util.Precision;
import org.apache.tez.common.JavaOptsChecker;
import org.apache.tez.dag.api.records.DAGProtos.AMPluginDescriptorProto;
import org.apache.tez.serviceplugins.api.ServicePluginsDescriptor;
@@ -1007,11 +1006,11 @@ public class TezClientUtils {
return javaOpts;
}
- if ((maxHeapFactor <= 0 && !Precision.equals(maxHeapFactor, -1, 0.01)) || maxHeapFactor >= 1) {
+ if ((maxHeapFactor <= 0 && Double.valueOf("-1") != maxHeapFactor) || maxHeapFactor >= 1) {
return javaOpts;
}
- if (Precision.equals(maxHeapFactor, -1, 0.01)) {
+ if (Double.valueOf("-1") == maxHeapFactor) {
maxHeapFactor = resource.getMemory() < TezConstants.TEZ_CONTAINER_SMALL_SLAB_BOUND_MB
? TezConstants.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_SMALL_SLAB
: TezConstants.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_LARGE_SLAB;
diff --git a/tez-dag/pom.xml b/tez-dag/pom.xml
index 8903385..c0dccc8 100644
--- a/tez-dag/pom.xml
+++ b/tez-dag/pom.xml
@@ -88,10 +88,6 @@
<artifactId>hadoop-yarn-server-web-proxy</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-math3</artifactId>
- </dependency>
- <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/DagAwareYarnTaskScheduler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/DagAwareYarnTaskScheduler.java
index d8b3397..29f350f 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/DagAwareYarnTaskScheduler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/DagAwareYarnTaskScheduler.java
@@ -22,7 +22,6 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.tez.common.Preconditions;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.commons.lang.mutable.MutableInt;
-import org.apache.commons.math3.random.RandomDataGenerator;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Time;
@@ -80,6 +79,7 @@ import java.util.concurrent.Future;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
/**
@@ -95,7 +95,6 @@ public class DagAwareYarnTaskScheduler extends TaskScheduler
private static final Logger LOG = LoggerFactory.getLogger(DagAwareYarnTaskScheduler.class);
private static final Comparator<HeldContainer> PREEMPT_ORDER_COMPARATOR = new PreemptOrderComparator();
- private final RandomDataGenerator random = new RandomDataGenerator();
private AMRMClientAsyncWrapper client;
private ScheduledExecutorService reuseExecutor;
private ResourceCalculator resourceCalculator;
@@ -1544,7 +1543,7 @@ public class DagAwareYarnTaskScheduler extends TaskScheduler
if (idleExpirationTimestamp == 0) {
if (idleContainerTimeoutMin > 0) {
idleExpirationTimestamp = now + (idleContainerTimeoutMin == idleContainerTimeoutMax ? idleContainerTimeoutMin
- : random.nextLong(idleContainerTimeoutMin, idleContainerTimeoutMax));
+ : ThreadLocalRandom.current().nextLong(idleContainerTimeoutMin, idleContainerTimeoutMax));
} else {
idleExpirationTimestamp = Long.MAX_VALUE;
}
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java
index b4aa561..6f51566 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java
@@ -32,6 +32,7 @@ import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.PriorityBlockingQueue;
+import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
@@ -44,7 +45,6 @@ import org.apache.tez.serviceplugins.api.TaskSchedulerContext.AppFinalStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.commons.lang.exception.ExceptionUtils;
-import org.apache.commons.math3.random.RandomDataGenerator;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.ExitUtil;
@@ -153,7 +153,6 @@ public class YarnTaskSchedulerService extends TaskScheduler
Set<ContainerId> sessionMinHeldContainers = Sets.newHashSet();
- RandomDataGenerator random = new RandomDataGenerator();
private final Configuration conf;
@VisibleForTesting
@@ -593,7 +592,7 @@ public class YarnTaskSchedulerService extends TaskScheduler
long expireTime = (startTime + idleContainerTimeoutMin);
if (idleContainerTimeoutMin != -1 && idleContainerTimeoutMin < idleContainerTimeoutMax) {
long expireTimeMax = startTime + idleContainerTimeoutMax;
- expireTime = random.nextLong(expireTime, expireTimeMax);
+ expireTime = ThreadLocalRandom.current().nextLong(expireTime, expireTimeMax);
}
return expireTime;