You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by je...@apache.org on 2020/02/24 22:27:15 UTC
[tez] branch branch-0.9 updated: TEZ-3664. Flaky tests due to
writing to /tmp directory
This is an automated email from the ASF dual-hosted git repository.
jeagles pushed a commit to branch branch-0.9
in repository https://gitbox.apache.org/repos/asf/tez.git
The following commit(s) were added to refs/heads/branch-0.9 by this push:
new c9d2ea7 TEZ-3664. Flaky tests due to writing to /tmp directory
c9d2ea7 is described below
commit c9d2ea78903e2da38e2deeb7e615e4cf82424606
Author: Jonathan Eagles <je...@apache.org>
AuthorDate: Mon Feb 24 16:25:00 2020 -0600
TEZ-3664. Flaky tests due to writing to /tmp directory
(cherry picked from commit 4c5db43044be74052e113314a6dc4c3ab9b1c72d)
---
pom.xml | 2 ++
.../test/java/org/apache/tez/client/TestTezClient.java | 4 ++++
.../java/org/apache/tez/client/TestTezClientUtils.java | 6 ++++++
.../java/org/apache/tez/common/TestTezCommonUtils.java | 8 ++++++--
.../java/org/apache/tez/dag/app/TestDAGAppMaster.java | 17 ++++++++---------
.../apache/tez/tests/TestExtServicesWithLocalMode.java | 4 ++++
.../apache/tez/mapreduce/output/TestMROutputLegacy.java | 11 +++++++----
.../apache/tez/mapreduce/output/TestMultiMROutput.java | 5 ++++-
.../org/apache/tez/auxservices/TestShuffleHandler.java | 15 +++++++++++++++
.../tez/dag/history/ats/acls/TestATSHistoryV15.java | 1 +
.../shuffle/impl/TestSimpleFetchedInputAllocator.java | 5 +++--
.../test/java/org/apache/tez/test/TestLocalMode.java | 10 +++++-----
.../apache/tez/test/TestTaskErrorsUsingLocalMode.java | 4 ++++
13 files changed, 69 insertions(+), 23 deletions(-)
diff --git a/pom.xml b/pom.xml
index f6bb760..890065d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -63,6 +63,7 @@
<maven-checkstyle-plugin.version>3.0.0</maven-checkstyle-plugin.version>
<checkstyle.version>8.16</checkstyle.version>
<dependency-check-maven.version>1.3.6</dependency-check-maven.version>
+ <test.build.data>${project.build.directory}/tmp</test.build.data>
</properties>
<scm>
<connection>${scm.url}</connection>
@@ -907,6 +908,7 @@
<MALLOC_ARENA_MAX>4</MALLOC_ARENA_MAX>
</environmentVariables>
<systemPropertyVariables>
+ <test.build.data>${test.build.data}</test.build.data>
<java.net.preferIPv4Stack>true</java.net.preferIPv4Stack>
<tez.hadoop.version>${hadoop.version}</tez.hadoop.version>
</systemPropertyVariables>
diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
index 0c297d3..91dce5e 100644
--- a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
+++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
@@ -18,6 +18,7 @@
package org.apache.tez.client;
+import java.io.File;
import java.io.IOException;
import java.net.InetAddress;
import java.nio.ByteBuffer;
@@ -108,6 +109,8 @@ import com.google.protobuf.RpcController;
public class TestTezClient {
static final long HARD_KILL_TIMEOUT = 1500L;
+ private static final File STAGING_DIR = new File(System.getProperty("test.build.data"),
+ TestTezClient.class.getName()).getAbsoluteFile();
class TezClientForTest extends TezClient {
TezYarnClient mockTezYarnClient;
@@ -163,6 +166,7 @@ public class TestTezClient {
}
conf.setBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, true);
conf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, isSession);
+ conf.set(TezConfiguration.TEZ_AM_STAGING_DIR, STAGING_DIR.getAbsolutePath());
TezClientForTest client = new TezClientForTest("test", conf, lrs, null);
ApplicationId appId1 = ApplicationId.newInstance(0, 1);
diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
index 581d722..0573bb9 100644
--- a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
+++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
@@ -81,6 +81,8 @@ import org.junit.Test;
public class TestTezClientUtils {
private static String TEST_ROOT_DIR = "target" + Path.SEPARATOR
+ TestTezClientUtils.class.getName() + "-tmpDir";
+ private static final File STAGING_DIR = new File(System.getProperty("test.build.data"),
+ TestTezClientUtils.class.getName()).getAbsoluteFile();
/**
*
*/
@@ -328,6 +330,7 @@ public class TestTezClientUtils {
// ApplicationSubmissionContext
public void testAppSubmissionContextForPriority() throws Exception {
TezConfiguration tezConf = new TezConfiguration();
+ tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, STAGING_DIR.getAbsolutePath());
int testpriority = 999;
ApplicationId appId = ApplicationId.newInstance(1000, 1);
Credentials credentials = new Credentials();
@@ -378,6 +381,7 @@ public class TestTezClientUtils {
public void testSessionTokenInAmClc() throws IOException, YarnException {
TezConfiguration tezConf = new TezConfiguration();
+ tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, STAGING_DIR.getAbsolutePath());
ApplicationId appId = ApplicationId.newInstance(1000, 1);
DAG dag = DAG.create("testdag");
@@ -415,6 +419,7 @@ public class TestTezClientUtils {
TezConfiguration tezConf = new TezConfiguration();
tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL, "WARN");
+ tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, STAGING_DIR.getAbsolutePath());
ApplicationId appId = ApplicationId.newInstance(1000, 1);
Credentials credentials = new Credentials();
@@ -455,6 +460,7 @@ public class TestTezClientUtils {
TezConfiguration tezConf = new TezConfiguration();
tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL,
"WARN;org.apache.hadoop.ipc=DEBUG;org.apache.hadoop.security=DEBUG");
+ tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, STAGING_DIR.getAbsolutePath());
ApplicationId appId = ApplicationId.newInstance(1000, 1);
Credentials credentials = new Credentials();
diff --git a/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java b/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java
index 3929c4b..d7bd397 100644
--- a/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java
+++ b/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java
@@ -18,6 +18,7 @@
package org.apache.tez.common;
+import java.io.File;
import java.io.IOException;
import java.util.Map;
@@ -43,6 +44,9 @@ import org.junit.Test;
public class TestTezCommonUtils {
private static final String STAGE_DIR = "/tmp/mystage";
+
+ private static final File LOCAL_STAGING_DIR = new File(System.getProperty("test.build.data"),
+ TestTezCommonUtils.class.getSimpleName()).getAbsoluteFile();
private static String RESOLVED_STAGE_DIR;
private static Configuration conf = new Configuration();;
private static String TEST_ROOT_DIR = "target" + Path.SEPARATOR
@@ -84,10 +88,10 @@ public class TestTezCommonUtils {
public void testTezBaseStagingPath() throws Exception {
Configuration localConf = new Configuration();
// Check if default works with localFS
- localConf.unset(TezConfiguration.TEZ_AM_STAGING_DIR);
+ localConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, LOCAL_STAGING_DIR.getAbsolutePath());
localConf.set("fs.defaultFS", "file:///");
Path stageDir = TezCommonUtils.getTezBaseStagingPath(localConf);
- Assert.assertEquals(stageDir.toString(), "file:" + TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT);
+ Assert.assertEquals("file:" + LOCAL_STAGING_DIR, stageDir.toString());
// check if user set something, indeed works
conf.set(TezConfiguration.TEZ_AM_STAGING_DIR, STAGE_DIR);
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestDAGAppMaster.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestDAGAppMaster.java
index c3a9af1..8cee1c4 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestDAGAppMaster.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestDAGAppMaster.java
@@ -93,15 +93,13 @@ public class TestDAGAppMaster {
private static final String CL_NAME = "CL";
private static final String TC_NAME = "TC";
private static final String CLASS_SUFFIX = "_CLASS";
- private static final File TEST_DIR = new File(
- System.getProperty("test.build.data",
- System.getProperty("java.io.tmpdir")),
- TestDAGAppMaster.class.getSimpleName()).getAbsoluteFile();
+ private static final File TEST_DIR = new File(System.getProperty("test.build.data"),
+ TestDAGAppMaster.class.getName()).getAbsoluteFile();
@Before
public void setup() {
FileUtil.fullyDelete(TEST_DIR);
- TEST_DIR.mkdir();
+ TEST_DIR.mkdirs();
}
@After
@@ -648,9 +646,10 @@ public class TestDAGAppMaster {
return creds;
}
- private static void stubSessionResources() throws IOException {
- FileOutputStream out = new FileOutputStream(
- new File(TEST_DIR, TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME));
+ private static void stubSessionResources(Configuration conf) throws IOException {
+ File file = new File(TEST_DIR, TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME);
+ conf.set(TezConfiguration.TEZ_AM_STAGING_DIR, TEST_DIR.getAbsolutePath());
+ FileOutputStream out = new FileOutputStream(file);
PlanLocalResourcesProto planProto = PlanLocalResourcesProto.getDefaultInstance();
planProto.writeDelimitedTo(out);
out.close();
@@ -658,7 +657,7 @@ public class TestDAGAppMaster {
@Override
public synchronized void serviceInit(Configuration conf) throws Exception {
- stubSessionResources();
+ stubSessionResources(conf);
conf.setBoolean(TezConfiguration.TEZ_AM_WEBSERVICE_ENABLE, false);
super.serviceInit(conf);
}
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExtServicesWithLocalMode.java b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExtServicesWithLocalMode.java
index 3d8c087..c0bfe76 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExtServicesWithLocalMode.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExtServicesWithLocalMode.java
@@ -20,6 +20,7 @@ package org.apache.tez.tests;
import static org.junit.Assert.assertEquals;
+import java.io.File;
import java.io.IOException;
import java.util.Map;
@@ -59,6 +60,8 @@ public class TestExtServicesWithLocalMode {
private static String TEST_ROOT_DIR =
"target" + Path.SEPARATOR + TestExtServicesWithLocalMode.class.getName()
+ "-tmpDir";
+ private static final String STAGING_DIR = new File(System.getProperty("test.build.data"),
+ TestExtServicesWithLocalMode.class.getName()).getAbsolutePath();
private static final Path SRC_DATA_DIR = new Path(TEST_ROOT_DIR + Path.SEPARATOR + "data");
private static final Path HASH_JOIN_EXPECTED_RESULT_PATH =
@@ -92,6 +95,7 @@ public class TestExtServicesWithLocalMode {
confForJobs.set(entry.getKey(), entry.getValue());
}
confForJobs.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
+ confForJobs.set(TezConfiguration.TEZ_AM_STAGING_DIR, STAGING_DIR.toString());
}
@AfterClass
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/output/TestMROutputLegacy.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/output/TestMROutputLegacy.java
index e4fa0ea..01b5c84 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/output/TestMROutputLegacy.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/output/TestMROutputLegacy.java
@@ -42,13 +42,16 @@ import org.apache.tez.mapreduce.hadoop.MRConfig;
import org.apache.tez.runtime.api.OutputContext;
import org.junit.Test;
+import java.io.File;
public class TestMROutputLegacy {
+ private static final File TEST_DIR = new File(System.getProperty("test.build.data"),
+ TestMROutputLegacy.class.getName()).getAbsoluteFile();
// simulate the behavior of translating MR to DAG using MR old API
@Test (timeout = 5000)
public void testOldAPI_MR() throws Exception {
- String outputPath = "/tmp/output";
+ String outputPath = TEST_DIR.getAbsolutePath();
JobConf conf = new JobConf();
conf.setOutputKeyClass(NullWritable.class);
conf.setOutputValueClass(Text.class);
@@ -79,7 +82,7 @@ public class TestMROutputLegacy {
// simulate the behavior of translating MR to DAG using MR new API
@Test (timeout = 5000)
public void testNewAPI_MR() throws Exception {
- String outputPath = "/tmp/output";
+ String outputPath = TEST_DIR.getAbsolutePath();
Job job = Job.getInstance();
job.setOutputKeyClass(NullWritable.class);
job.setOutputValueClass(Text.class);
@@ -111,7 +114,7 @@ public class TestMROutputLegacy {
// simulate the behavior of translating Mapper-only job to DAG using MR old API
@Test (timeout = 5000)
public void testOldAPI_MapperOnly() throws Exception {
- String outputPath = "/tmp/output";
+ String outputPath = TEST_DIR.getAbsolutePath();
JobConf conf = new JobConf();
conf.setOutputKeyClass(NullWritable.class);
conf.setOutputValueClass(Text.class);
@@ -142,7 +145,7 @@ public class TestMROutputLegacy {
//simulate the behavior of translating mapper-only job to DAG using MR new API
@Test (timeout = 5000)
public void testNewAPI_MapperOnly() throws Exception {
- String outputPath = "/tmp/output";
+ String outputPath = TEST_DIR.getAbsolutePath();
Job job = Job.getInstance();
job.setOutputKeyClass(NullWritable.class);
job.setOutputValueClass(Text.class);
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/output/TestMultiMROutput.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/output/TestMultiMROutput.java
index 3618e40..c8eca16 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/output/TestMultiMROutput.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/output/TestMultiMROutput.java
@@ -39,10 +39,13 @@ import org.apache.tez.runtime.api.OutputStatisticsReporter;
import org.junit.Assert;
import org.junit.Test;
+import java.io.File;
import java.io.IOException;
public class TestMultiMROutput {
+ private static final File TEST_DIR = new File(System.getProperty("test.build.data"),
+ TestMultiMROutput.class.getName()).getAbsoluteFile();
@Test(timeout = 5000)
public void testNewAPI_TextOutputFormat() throws Exception {
@@ -176,7 +179,7 @@ public class TestMultiMROutput {
private MultiMROutput createMROutputs(Class outputFormat,
boolean isMapper, boolean useLazyOutputFormat)
throws InterruptedException, IOException {
- String outputPath = "/tmp/output";
+ String outputPath = TEST_DIR.getAbsolutePath();
JobConf conf = new JobConf();
conf.setBoolean(MRConfig.IS_MAP_PROCESSOR, isMapper);
conf.setOutputKeyClass(Text.class);
diff --git a/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestShuffleHandler.java b/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestShuffleHandler.java
index b622d98..b75a160 100644
--- a/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestShuffleHandler.java
+++ b/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestShuffleHandler.java
@@ -102,6 +102,9 @@ import org.slf4j.LoggerFactory;
public class TestShuffleHandler {
static final long MiB = 1024 * 1024;
private static final Logger LOG = LoggerFactory.getLogger(TestShuffleHandler.class);
+ private static final File TEST_DIR = new File(System.getProperty("test.build.data"),
+ TestShuffleHandler.class.getName()).getAbsoluteFile();
+ private static final String HADOOP_TMP_DIR = "hadoop.tmp.dir";
class MockShuffleHandler extends org.apache.tez.auxservices.ShuffleHandler {
@Override
protected Shuffle getShuffle(final Configuration conf) {
@@ -230,6 +233,7 @@ public class TestShuffleHandler {
public void testClientClosesConnection() throws Exception {
final ArrayList<Throwable> failures = new ArrayList<Throwable>(1);
Configuration conf = new Configuration();
+ conf.set(HADOOP_TMP_DIR, TEST_DIR.getAbsolutePath());
conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
ShuffleHandler shuffleHandler = new ShuffleHandler() {
@Override
@@ -335,6 +339,7 @@ public class TestShuffleHandler {
public void testKeepAlive() throws Exception {
final ArrayList<Throwable> failures = new ArrayList<Throwable>(1);
Configuration conf = new Configuration();
+ conf.set(HADOOP_TMP_DIR, TEST_DIR.getAbsolutePath());
conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
conf.setBoolean(ShuffleHandler.SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED, true);
// try setting to -ve keep alive timeout.
@@ -484,6 +489,7 @@ public class TestShuffleHandler {
@Test
public void testSocketKeepAlive() throws Exception {
Configuration conf = new Configuration();
+ conf.set(HADOOP_TMP_DIR, TEST_DIR.getAbsolutePath());
conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
conf.setBoolean(ShuffleHandler.SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED, true);
// try setting to -ve keep alive timeout.
@@ -527,6 +533,7 @@ public class TestShuffleHandler {
public void testIncompatibleShuffleVersion() throws Exception {
final int failureNum = 3;
Configuration conf = new Configuration();
+ conf.set(HADOOP_TMP_DIR, TEST_DIR.getAbsolutePath());
conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
ShuffleHandler shuffleHandler = new ShuffleHandler();
shuffleHandler.init(conf);
@@ -561,6 +568,7 @@ public class TestShuffleHandler {
public void testMaxConnections() throws Exception {
Configuration conf = new Configuration();
+ conf.set(HADOOP_TMP_DIR, TEST_DIR.getAbsolutePath());
conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3);
ShuffleHandler shuffleHandler = new ShuffleHandler() {
@@ -665,6 +673,7 @@ public class TestShuffleHandler {
@Test(timeout = 10000)
public void testRangedFetch() throws IOException {
Configuration conf = new Configuration();
+ conf.set(HADOOP_TMP_DIR, TEST_DIR.getAbsolutePath());
conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3);
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
@@ -766,6 +775,7 @@ public class TestShuffleHandler {
// This will run only in NativeIO is enabled as SecureIOUtils need it
assumeTrue(NativeIO.isAvailable());
Configuration conf = new Configuration();
+ conf.set(HADOOP_TMP_DIR, TEST_DIR.getAbsolutePath());
conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3);
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
@@ -899,6 +909,7 @@ public class TestShuffleHandler {
System.getProperty("java.io.tmpdir")),
TestShuffleHandler.class.getName());
Configuration conf = new Configuration();
+ conf.set(HADOOP_TMP_DIR, TEST_DIR.getAbsolutePath());
conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3);
ShuffleHandler shuffle = new ShuffleHandler();
@@ -966,6 +977,7 @@ public class TestShuffleHandler {
System.getProperty("java.io.tmpdir")),
TestShuffleHandler.class.getName());
Configuration conf = new Configuration();
+ conf.set(HADOOP_TMP_DIR, TEST_DIR.getAbsolutePath());
conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3);
ShuffleHandler shuffle = new ShuffleHandler();
@@ -1072,6 +1084,7 @@ public class TestShuffleHandler {
public void testGetMapOutputInfo() throws Exception {
final ArrayList<Throwable> failures = new ArrayList<Throwable>(1);
Configuration conf = new Configuration();
+ conf.set(HADOOP_TMP_DIR, TEST_DIR.getAbsolutePath());
conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3);
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
@@ -1176,6 +1189,7 @@ public class TestShuffleHandler {
public void testDagDelete() throws Exception {
final ArrayList<Throwable> failures = new ArrayList<Throwable>(1);
Configuration conf = new Configuration();
+ conf.set(HADOOP_TMP_DIR, TEST_DIR.getAbsolutePath());
conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3);
conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
@@ -1286,6 +1300,7 @@ public class TestShuffleHandler {
final ShuffleHandler sh = new MockShuffleHandler();
Configuration conf = new Configuration();
+ conf.set(HADOOP_TMP_DIR, TEST_DIR.getAbsolutePath());
// The Shuffle handler port associated with the service is bound to but not used.
conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
sh.init(conf);
diff --git a/tez-plugins/tez-yarn-timeline-history-with-fs/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryV15.java b/tez-plugins/tez-yarn-timeline-history-with-fs/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryV15.java
index 54abd44..f49e588 100644
--- a/tez-plugins/tez-yarn-timeline-history-with-fs/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryV15.java
+++ b/tez-plugins/tez-yarn-timeline-history-with-fs/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryV15.java
@@ -98,6 +98,7 @@ public class TestATSHistoryV15 {
1, 1, 1, true);
Configuration conf = new Configuration();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ conf.set(YarnConfiguration.TIMELINE_SERVICE_LEVELDB_PATH, TEST_ROOT_DIR);
conf.set("fs.defaultFS", remoteFs.getUri().toString()); // use HDFS
conf.setInt("yarn.nodemanager.delete.debug-delay-sec", 20000);
atsActivePath = new Path("/tmp/ats/active/" + random.nextInt(100000));
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestSimpleFetchedInputAllocator.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestSimpleFetchedInputAllocator.java
index 1b63b17..01faa5d 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestSimpleFetchedInputAllocator.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestSimpleFetchedInputAllocator.java
@@ -20,6 +20,7 @@ package org.apache.tez.runtime.library.common.shuffle.impl;
import static org.junit.Assert.assertEquals;
+import java.io.File;
import java.io.IOException;
import java.util.UUID;
@@ -38,7 +39,7 @@ public class TestSimpleFetchedInputAllocator {
@Test(timeout = 5000)
public void testInMemAllocation() throws IOException {
- String localDirs = "/tmp/" + this.getClass().getName();
+ File localDirs = new File(System.getProperty("test.build.data", "/tmp"), this.getClass().getName());
Configuration conf = new Configuration();
long jvmMax = Runtime.getRuntime().maxMemory();
@@ -47,7 +48,7 @@ public class TestSimpleFetchedInputAllocator {
float bufferPercent = 0.1f;
conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, bufferPercent);
conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 1.0f);
- conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, localDirs);
+ conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, localDirs.getAbsolutePath());
long inMemThreshold = (long) (bufferPercent * jvmMax);
LOG.info("InMemThreshold: " + inMemThreshold);
diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestLocalMode.java b/tez-tests/src/test/java/org/apache/tez/test/TestLocalMode.java
index ffc67fe..e6ef8c9 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/TestLocalMode.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/TestLocalMode.java
@@ -61,9 +61,8 @@ import static org.junit.Assert.*;
@RunWith(Parameterized.class)
public class TestLocalMode {
- private static final File TEST_DIR = new File(
- System.getProperty("test.build.data",
- System.getProperty("java.io.tmpdir")), "TestLocalMode-tez-localmode");
+ private static final File STAGING_DIR = new File(System.getProperty("test.build.data"),
+ TestLocalMode.class.getName());
private static MiniDFSCluster dfsCluster;
private static FileSystem remoteFs;
@@ -111,6 +110,7 @@ public class TestLocalMode {
} else {
conf.set("fs.defaultFS", "file:///");
}
+ conf.set(TezConfiguration.TEZ_AM_STAGING_DIR, STAGING_DIR.getAbsolutePath());
conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true);
return conf;
}
@@ -261,9 +261,9 @@ public class TestLocalMode {
//create inputs and outputs
FileSystem fs = FileSystem.get(tezConf);
for(int i = 0; i < dags; i++) {
- inputPaths[i] = new Path(TEST_DIR.getAbsolutePath(),"in-"+i).toString();
+ inputPaths[i] = new Path(STAGING_DIR.getAbsolutePath(), "in-" + i).toString();
createInputFile(fs, inputPaths[i]);
- outputPaths[i] = new Path(TEST_DIR.getAbsolutePath(),"out-"+i).toString();
+ outputPaths[i] = new Path(STAGING_DIR.getAbsolutePath(), "out-" + i).toString();
}
//start testing
diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestTaskErrorsUsingLocalMode.java b/tez-tests/src/test/java/org/apache/tez/test/TestTaskErrorsUsingLocalMode.java
index d622698..d7d2dd9 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/TestTaskErrorsUsingLocalMode.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/TestTaskErrorsUsingLocalMode.java
@@ -16,6 +16,7 @@ package org.apache.tez.test;
import static org.junit.Assert.assertEquals;
+import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.Map;
@@ -44,6 +45,8 @@ public class TestTaskErrorsUsingLocalMode {
private static final Logger LOG = LoggerFactory.getLogger(TestTaskErrorsUsingLocalMode.class);
private static final String VERTEX_NAME = "vertex1";
+ private static final File STAGING_DIR = new File(System.getProperty("test.build.data"),
+ TestTaskErrorsUsingLocalMode.class.getName()).getAbsoluteFile();
@Test(timeout = 20000)
@@ -123,6 +126,7 @@ public class TestTaskErrorsUsingLocalMode {
TezConfiguration tezConf1 = new TezConfiguration();
tezConf1.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
tezConf1.set("fs.defaultFS", "file:///");
+ tezConf1.set(TezConfiguration.TEZ_AM_STAGING_DIR, STAGING_DIR.getAbsolutePath());
tezConf1.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true);
tezConf1.setLong(TezConfiguration.TEZ_AM_SLEEP_TIME_BEFORE_EXIT_MILLIS, 500);
TezClient tezClient1 = TezClient.create(name, tezConf1, true);