You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ab...@apache.org on 2020/08/26 18:03:37 UTC
[tez] branch master updated: TEZ-4175: Consider removing YarnConfiguration where it's possible (László Bodor reviewed by Rajesh Balamohan, Mustafa Iman, Ashutosh Chauhan)
This is an automated email from the ASF dual-hosted git repository.
abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tez.git
The following commit(s) were added to refs/heads/master by this push:
new 0e17787 TEZ-4175: Consider removing YarnConfiguration where it's possible (László Bodor reviewed by Rajesh Balamohan, Mustafa Iman, Ashutosh Chauhan)
0e17787 is described below
commit 0e17787c3547a0480a7ffb2069a4c62a72783cb9
Author: László Bodor <bo...@gmail.com>
AuthorDate: Wed Aug 26 19:58:35 2020 +0200
TEZ-4175: Consider removing YarnConfiguration where it's possible (László Bodor reviewed by Rajesh Balamohan, Mustafa Iman, Ashutosh Chauhan)
Signed-off-by: Laszlo Bodor <bo...@gmail.com>
---
.../org/apache/tez/client/AMConfiguration.java | 10 ---------
.../org/apache/tez/client/FrameworkClient.java | 5 +----
.../main/java/org/apache/tez/client/TezClient.java | 24 ++++++++--------------
.../java/org/apache/tez/client/TezYarnClient.java | 4 ++--
.../apache/tez/dag/api/client/DAGClientImpl.java | 6 ++----
.../java/org/apache/tez/client/TestTezClient.java | 3 +--
.../tez/dag/api/client/rpc/TestDAGClient.java | 18 +++++-----------
.../java/org/apache/tez/client/LocalClient.java | 4 +---
.../java/org/apache/tez/dag/app/DAGAppMaster.java | 4 +---
.../apache/tez/dag/app/rm/TestContainerReuse.java | 23 ++++++++++-----------
.../apache/tez/mapreduce/client/YARNRunner.java | 2 +-
.../tez/mapreduce/hadoop/TestMRInputHelpers.java | 3 +--
.../apache/tez/test/FaultToleranceTestRunner.java | 9 ++++----
13 files changed, 38 insertions(+), 77 deletions(-)
diff --git a/tez-api/src/main/java/org/apache/tez/client/AMConfiguration.java b/tez-api/src/main/java/org/apache/tez/client/AMConfiguration.java
index 238b155..1b61c45 100644
--- a/tez-api/src/main/java/org/apache/tez/client/AMConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/client/AMConfiguration.java
@@ -23,7 +23,6 @@ import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.tez.dag.api.TezConfiguration;
import com.google.common.collect.Maps;
@@ -34,7 +33,6 @@ class AMConfiguration {
private Map<String, LocalResource> amLocalResources = Maps.newHashMap();
private TezConfiguration tezConf;
private Credentials credentials;
- private YarnConfiguration yarnConfig;
private LocalResource binaryConfLRsrc;
AMConfiguration(TezConfiguration tezConf, Map<String, LocalResource> localResources,
@@ -64,10 +62,6 @@ class AMConfiguration {
void setTezConfiguration(TezConfiguration tezConf) {
this.tezConf = tezConf;
}
-
- void setYarnConfiguration(YarnConfiguration yarnConf) {
- this.yarnConfig = yarnConf;
- }
String getQueueName() {
return this.tezConf.get(TezConfiguration.TEZ_QUEUE_NAME);
@@ -81,10 +75,6 @@ class AMConfiguration {
return tezConf;
}
- YarnConfiguration getYarnConfiguration() {
- return yarnConfig;
- }
-
Credentials getCredentials() {
return credentials;
}
diff --git a/tez-api/src/main/java/org/apache/tez/client/FrameworkClient.java b/tez-api/src/main/java/org/apache/tez/client/FrameworkClient.java
index b3e084c..7c60ec1 100644
--- a/tez-api/src/main/java/org/apache/tez/client/FrameworkClient.java
+++ b/tez-api/src/main/java/org/apache/tez/client/FrameworkClient.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.api.YarnClientApplication;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.tez.common.ReflectionUtils;
import org.apache.tez.dag.api.TezConfiguration;
@@ -58,10 +57,8 @@ public abstract class FrameworkClient {
*
* @param tezConf the {@link org.apache.tez.dag.api.TezConfiguration} instance being used by the
* cluster
- * @param yarnConf the {@link org.apache.hadoop.yarn.conf.YarnConfiguration} instance being used
- * by the cluster
*/
- public abstract void init(TezConfiguration tezConf, YarnConfiguration yarnConf);
+ public abstract void init(TezConfiguration tezConf);
public abstract void start();
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClient.java b/tez-api/src/main/java/org/apache/tez/client/TezClient.java
index 4fb37dc..fbe3509 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClient.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClient.java
@@ -57,7 +57,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.util.Time;
@@ -384,7 +383,6 @@ public class TezClient {
* @throws IOException
*/
public synchronized void start() throws TezException, IOException {
- amConfig.setYarnConfiguration(new YarnConfiguration(amConfig.getTezConfiguration()));
startFrameworkClient();
setupJavaOptsChecker();
@@ -434,7 +432,6 @@ public class TezClient {
*/
public synchronized TezClient getClient(ApplicationId appId) throws TezException, IOException {
sessionAppId = appId;
- amConfig.setYarnConfiguration(new YarnConfiguration(amConfig.getTezConfiguration()));
startFrameworkClient();
setupJavaOptsChecker();
@@ -466,7 +463,7 @@ public class TezClient {
private void startFrameworkClient() {
frameworkClient = createFrameworkClient();
- frameworkClient.init(amConfig.getTezConfiguration(), amConfig.getYarnConfiguration());
+ frameworkClient.init(amConfig.getTezConfiguration());
frameworkClient.start();
}
@@ -716,7 +713,6 @@ public class TezClient {
+ ", dagName=" + dag.getName());
return new DAGClientImpl(sessionAppId, dagId,
amConfig.getTezConfiguration(),
- amConfig.getYarnConfiguration(),
frameworkClient, getUgi());
}
@@ -1068,7 +1064,7 @@ public class TezClient {
protected DAGClientAMProtocolBlockingPB getAMProxy(ApplicationId appId)
throws TezException, IOException {
return TezClientUtils.getAMProxy(
- frameworkClient, amConfig.getYarnConfiguration(), appId, getUgi());
+ frameworkClient, amConfig.getTezConfiguration(), appId, getUgi());
}
private DAGClientAMProtocolBlockingPB waitForProxy()
@@ -1146,8 +1142,7 @@ public class TezClient {
}
// wait for dag in non-session mode to start running, so that we can start to getDAGStatus
waitNonSessionTillReady();
- return getDAGClient(appId, amConfig.getTezConfiguration(), amConfig.getYarnConfiguration(),
- frameworkClient, getUgi());
+ return getDAGClient(appId, amConfig.getTezConfiguration(), frameworkClient, getUgi());
}
private ApplicationId createApplication() throws TezException, IOException {
@@ -1170,20 +1165,17 @@ public class TezClient {
}
@Private
- static DAGClient getDAGClient(ApplicationId appId, TezConfiguration tezConf, YarnConfiguration
- yarnConf, FrameworkClient frameworkClient, UserGroupInformation ugi)
- throws IOException, TezException {
- return new DAGClientImpl(appId, getDefaultTezDAGID(appId), tezConf,
- yarnConf, frameworkClient, ugi);
+ static DAGClient getDAGClient(ApplicationId appId, TezConfiguration tezConf,
+ FrameworkClient frameworkClient, UserGroupInformation ugi) throws IOException, TezException {
+ return new DAGClientImpl(appId, getDefaultTezDAGID(appId), tezConf, frameworkClient, ugi);
}
@Private // Used only for MapReduce compatibility code
static DAGClient getDAGClient(ApplicationId appId, TezConfiguration tezConf,
- FrameworkClient frameworkClient)
- throws IOException, TezException {
+ FrameworkClient frameworkClient) throws IOException, TezException {
UserGroupInformation ugi =
UserGroupInformation.createRemoteUser(UserGroupInformation.getCurrentUser().getUserName());
- return getDAGClient(appId, tezConf, new YarnConfiguration(tezConf), frameworkClient, ugi);
+ return getDAGClient(appId, tezConf, frameworkClient, ugi);
}
// DO NOT CHANGE THIS. This code is replicated from TezDAGID.java
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java b/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java
index 2a0c79a..f89664d 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java
@@ -45,8 +45,8 @@ public class TezYarnClient extends FrameworkClient {
}
@Override
- public void init(TezConfiguration tezConf, YarnConfiguration yarnConf) {
- yarnClient.init(yarnConf);
+ public void init(TezConfiguration tezConf) {
+ yarnClient.init(new YarnConfiguration(tezConf));
}
@Override
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientImpl.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientImpl.java
index 0b899fd..01a10b2 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientImpl.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientImpl.java
@@ -40,7 +40,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.tez.client.FrameworkClient;
import org.apache.tez.common.counters.TezCounters;
@@ -80,8 +79,7 @@ public class DAGClientImpl extends DAGClient {
private boolean cleanupFrameworkClient;
public DAGClientImpl(ApplicationId appId, String dagId, TezConfiguration conf,
- YarnConfiguration yarnConf, @Nullable FrameworkClient frameworkClient,
- UserGroupInformation ugi) {
+ @Nullable FrameworkClient frameworkClient, UserGroupInformation ugi) {
this.appId = appId;
this.dagId = dagId;
this.conf = conf;
@@ -89,7 +87,7 @@ public class DAGClientImpl extends DAGClient {
this.frameworkClient = frameworkClient;
} else {
this.frameworkClient = FrameworkClient.createFrameworkClient(conf);
- this.frameworkClient.init(conf, yarnConf);
+ this.frameworkClient.init(conf);
this.frameworkClient.start();
cleanupFrameworkClient = true;
}
diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
index 91dce5e..7316452 100644
--- a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
+++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
@@ -67,7 +67,6 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.tez.common.counters.LimitExceededException;
@@ -889,7 +888,7 @@ public class TestTezClient {
when(yarnClient.createApplication().getNewApplicationResponse().getApplicationId()).thenReturn(appId1);
when(yarnClient.getApplicationReport(appId1)).thenReturn(mockReport);
TezYarnClient tezClient = new TezYarnClient(yarnClient);
- tezClient.init(new TezConfiguration(false), new YarnConfiguration());
+ tezClient.init(new TezConfiguration(false));
try {
tezClient.getApplicationReport(appId1);
fail("getApplicationReport should have thrown");
diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClient.java b/tez-api/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClient.java
index 57087cb..211baf2 100644
--- a/tez-api/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClient.java
+++ b/tez-api/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClient.java
@@ -35,7 +35,6 @@ import java.util.Set;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.tez.client.FrameworkClient;
import org.apache.tez.dag.api.TezConfiguration;
@@ -204,8 +203,7 @@ public class TestDAGClient {
.thenReturn(GetVertexStatusResponseProto.newBuilder().setVertexStatus(vertexStatusProtoWithCounters).build());
TezConfiguration tezConf = new TezConfiguration();
- YarnConfiguration yarnConf = new YarnConfiguration(tezConf);
- dagClient = new DAGClientImpl(mockAppId, dagIdStr, tezConf, yarnConf, null,
+ dagClient = new DAGClientImpl(mockAppId, dagIdStr, tezConf, null,
UserGroupInformation.getCurrentUser());
DAGClientRPCImpl realClient = (DAGClientRPCImpl)((DAGClientImpl)dagClient).getRealClient();
realClient.appReport = mockAppReport;
@@ -342,17 +340,14 @@ public class TestDAGClient {
TezConfiguration tezConf = new TezConfiguration();
tezConf.setLong(TezConfiguration.TEZ_DAG_STATUS_POLLINTERVAL_MS, 800l);
- YarnConfiguration yarnConf = new YarnConfiguration(tezConf);
- DAGClientImplForTest dagClient = new DAGClientImplForTest(mockAppId, dagIdStr, tezConf,
- yarnConf,null);
+ DAGClientImplForTest dagClient = new DAGClientImplForTest(mockAppId, dagIdStr, tezConf, null);
DAGClientRPCImplForTest dagClientRpc =
new DAGClientRPCImplForTest(mockAppId, dagIdStr, tezConf, null);
dagClient.setRealClient(dagClientRpc);
DAGStatus dagStatus;
-
// Fetch from RM. AM not up yet.
dagClientRpc.setAMProxy(null);
DAGStatus rmDagStatus =
@@ -426,14 +421,12 @@ public class TestDAGClient {
String loggingClass, boolean amHistoryLoggingEnabled,
boolean dagHistoryLoggingEnabled) throws IOException {
TezConfiguration tezConf = new TezConfiguration();
- YarnConfiguration yarnConf = new YarnConfiguration(tezConf);
tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, loggingClass);
tezConf.setBoolean(TezConfiguration.TEZ_AM_HISTORY_LOGGING_ENABLED, amHistoryLoggingEnabled);
tezConf.setBoolean(TezConfiguration.TEZ_DAG_HISTORY_LOGGING_ENABLED, dagHistoryLoggingEnabled);
- DAGClientImplForTest dagClient = new DAGClientImplForTest(appId, dagIdStr, tezConf,
- yarnConf,null);
+ DAGClientImplForTest dagClient = new DAGClientImplForTest(appId, dagIdStr, tezConf, null);
assertEquals(expected, dagClient.getIsATSEnabled());
}
@@ -478,9 +471,8 @@ public class TestDAGClient {
int numGetStatusViaRmInvocations = 0;
public DAGClientImplForTest(ApplicationId appId, String dagId, TezConfiguration conf,
- YarnConfiguration yarnConf,
@Nullable FrameworkClient frameworkClient) throws IOException {
- super(appId, dagId, conf, yarnConf, frameworkClient, UserGroupInformation.getCurrentUser());
+ super(appId, dagId, conf, frameworkClient, UserGroupInformation.getCurrentUser());
}
private void setRealClient(DAGClientRPCImplForTest dagClientRpcImplForTest) {
@@ -518,7 +510,7 @@ public class TestDAGClient {
ServiceException {
DAGClientAMProtocolBlockingPB mock = mock(DAGClientAMProtocolBlockingPB.class);
- doAnswer(new Answer() {
+ doAnswer(new Answer<Object>() {
@Override
public Object answer(InvocationOnMock invocation) throws Throwable {
GetDAGStatusRequestProto request = (GetDAGStatusRequestProto) invocation.getArguments()[1];
diff --git a/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java b/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java
index 9006971..5a6bb9a 100644
--- a/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java
+++ b/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java
@@ -18,14 +18,12 @@
package org.apache.tez.client;
-import java.io.File;
import java.io.IOException;
import java.net.InetAddress;
import java.nio.ByteBuffer;
import java.util.List;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
@@ -81,7 +79,7 @@ public class LocalClient extends FrameworkClient {
}
@Override
- public void init(TezConfiguration tezConf, YarnConfiguration yarnConf) {
+ public void init(TezConfiguration tezConf) {
this.conf = tezConf;
// Tez libs already in the client's classpath
this.conf.setBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, true);
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 5400668..f4a8923 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -105,7 +105,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.Event;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -2390,8 +2389,7 @@ public class DAGAppMaster extends AbstractService {
+ ", localDirs=" + System.getenv(Environment.LOCAL_DIRS.name())
+ ", logDirs=" + System.getenv(Environment.LOG_DIRS.name()));
- // TODO Does this really need to be a YarnConfiguration ?
- Configuration conf = new Configuration(new YarnConfiguration());
+ Configuration conf = new Configuration();
ConfigurationProto confProto =
TezUtilsInternal.readUserSpecifiedTezConfiguration(System.getenv(Environment.PWD.name()));
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
index 4e29dd5..92ea8a8 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
@@ -55,7 +55,6 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.tez.common.MockDNSToSwitchMapping;
import org.apache.tez.dag.api.InputDescriptor;
import org.apache.tez.dag.api.OutputDescriptor;
@@ -114,7 +113,7 @@ public class TestContainerReuse {
public void testDelayedReuseContainerBecomesAvailable()
throws IOException, InterruptedException, ExecutionException {
LOG.info("Test testDelayedReuseContainerBecomesAvailable");
- Configuration conf = new Configuration(new YarnConfiguration());
+ Configuration conf = new Configuration();
conf.setBoolean(
TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, true);
conf.setBoolean(
@@ -249,7 +248,7 @@ public class TestContainerReuse {
public void testDelayedReuseContainerNotAvailable()
throws IOException, InterruptedException, ExecutionException {
LOG.info("Test testDelayedReuseContainerNotAvailable");
- Configuration conf = new Configuration(new YarnConfiguration());
+ Configuration conf = new Configuration();
conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, true);
conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED, false);
conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED, false);
@@ -351,7 +350,7 @@ public class TestContainerReuse {
@Test(timeout = 10000l)
public void testSimpleReuse() throws IOException, InterruptedException, ExecutionException {
LOG.info("Test testSimpleReuse");
- Configuration tezConf = new Configuration(new YarnConfiguration());
+ Configuration tezConf = new Configuration();
tezConf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, true);
tezConf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED, true);
tezConf.setLong(TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS, 0);
@@ -492,7 +491,7 @@ public class TestContainerReuse {
@Test(timeout = 10000l)
public void testReuseWithTaskSpecificLaunchCmdOption() throws IOException, InterruptedException, ExecutionException {
LOG.info("Test testReuseWithTaskSpecificLaunchCmdOption");
- Configuration tezConf = new Configuration(new YarnConfiguration());
+ Configuration tezConf = new Configuration();
tezConf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, true);
tezConf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED, true);
tezConf.setLong(TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS, 0);
@@ -682,7 +681,7 @@ public class TestContainerReuse {
public void testReuseNonLocalRequest()
throws IOException, InterruptedException, ExecutionException {
LOG.info("Test testReuseNonLocalRequest");
- Configuration tezConf = new Configuration(new YarnConfiguration());
+ Configuration tezConf = new Configuration();
tezConf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, true);
tezConf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED, true);
tezConf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED, true);
@@ -803,7 +802,7 @@ public class TestContainerReuse {
public void testReuseAcrossVertices()
throws IOException, InterruptedException, ExecutionException {
LOG.info("Test testReuseAcrossVertices");
- Configuration tezConf = new Configuration(new YarnConfiguration());
+ Configuration tezConf = new Configuration();
tezConf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, true);
tezConf.setLong(
TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS, 1l);
@@ -923,7 +922,7 @@ public class TestContainerReuse {
@Test(timeout = 30000l)
public void testReuseLocalResourcesChanged() throws IOException, InterruptedException, ExecutionException {
LOG.info("Test testReuseLocalResourcesChanged");
- Configuration tezConf = new Configuration(new YarnConfiguration());
+ Configuration tezConf = new Configuration();
tezConf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, true);
tezConf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED, true);
tezConf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED, true);
@@ -1080,7 +1079,7 @@ public class TestContainerReuse {
@Test(timeout = 30000l)
public void testReuseConflictLocalResources() throws IOException, InterruptedException, ExecutionException {
LOG.info("Test testReuseLocalResourcesChanged");
- Configuration tezConf = new Configuration(new YarnConfiguration());
+ Configuration tezConf = new Configuration();
tezConf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, true);
tezConf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED, true);
tezConf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED, true);
@@ -1317,7 +1316,7 @@ public class TestContainerReuse {
public void testAssignmentOnShutdown()
throws IOException, InterruptedException, ExecutionException {
LOG.info("Test testAssignmentOnShutdown");
- Configuration tezConf = new Configuration(new YarnConfiguration());
+ Configuration tezConf = new Configuration();
tezConf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, false);
tezConf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED, true);
tezConf.setLong(TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS, 0);
@@ -1384,7 +1383,7 @@ public class TestContainerReuse {
@Test(timeout=5000)
public void testDifferentResourceContainerReuse() throws Exception {
- Configuration tezConf = new Configuration(new YarnConfiguration());
+ Configuration tezConf = new Configuration();
tezConf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, true);
tezConf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED, true);
tezConf.setLong(TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS, 0);
@@ -1520,7 +1519,7 @@ public class TestContainerReuse {
@Test(timeout=5000)
public void testEnvironmentVarsContainerReuse() throws Exception {
- Configuration tezConf = new Configuration(new YarnConfiguration());
+ Configuration tezConf = new Configuration();
tezConf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, true);
tezConf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED, true);
tezConf.setLong(TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS, 0);
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
index 00a68cd..5a7d754 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
@@ -148,7 +148,7 @@ public class YARNRunner implements ClientProtocol {
* @param conf the configuration object for the client
*/
public YARNRunner(Configuration conf) {
- this(conf, new ResourceMgrDelegate(new YarnConfiguration(conf)));
+ this(conf, new ResourceMgrDelegate(new YarnConfiguration(conf)));
}
/**
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRInputHelpers.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRInputHelpers.java
index 88cc4a5..11b1271 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRInputHelpers.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRInputHelpers.java
@@ -40,7 +40,6 @@ import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.split.JobSplit;
import org.apache.hadoop.mapreduce.split.SplitMetaInfoReader;
import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.tez.dag.api.DataSourceDescriptor;
import org.apache.tez.dag.api.TaskLocationHint;
import org.junit.Assert;
@@ -71,7 +70,7 @@ public class TestMRInputHelpers {
throw new RuntimeException("problem starting mini dfs cluster", io);
}
- Configuration testConf = new YarnConfiguration(
+ Configuration testConf = new Configuration(
dfsCluster.getFileSystem().getConf());
diff --git a/tez-tests/src/test/java/org/apache/tez/test/FaultToleranceTestRunner.java b/tez-tests/src/test/java/org/apache/tez/test/FaultToleranceTestRunner.java
index 94242a7..9305711 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/FaultToleranceTestRunner.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/FaultToleranceTestRunner.java
@@ -25,7 +25,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.tez.client.TezClientUtils;
import org.apache.tez.client.TezClient;
import org.apache.tez.dag.api.DAG;
@@ -52,10 +51,10 @@ public class FaultToleranceTestRunner {
void setup() throws Exception {
TezConfiguration tezConf = null;
- if (conf == null ) {
- tezConf = new TezConfiguration(new YarnConfiguration());
- }else {
- tezConf = new TezConfiguration(new YarnConfiguration(this.conf));
+ if (conf == null) {
+ tezConf = new TezConfiguration();
+ } else {
+ tezConf = new TezConfiguration(this.conf);
}
FileSystem defaultFs = FileSystem.get(tezConf);