You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ab...@apache.org on 2020/10/13 05:50:16 UTC
[tez] branch branch-0.9 updated: TEZ-4229: Improve TezLocalCacheManager to use configured root directory (László Bodor reviewed by Panagiotis Garefalakis, Ashutosh Chauhan)
This is an automated email from the ASF dual-hosted git repository.
abstractdog pushed a commit to branch branch-0.9
in repository https://gitbox.apache.org/repos/asf/tez.git
The following commit(s) were added to refs/heads/branch-0.9 by this push:
new f4979c8 TEZ-4229: Improve TezLocalCacheManager to use configured root directory (László Bodor reviewed by Panagiotis Garefalakis, Ashutosh Chauhan)
f4979c8 is described below
commit f4979c80f7970daaf9e8398fab35d96e6f05a5c4
Author: László Bodor <bo...@gmail.com>
AuthorDate: Tue Oct 13 07:48:36 2020 +0200
TEZ-4229: Improve TezLocalCacheManager to use configured root directory (László Bodor reviewed by Panagiotis Garefalakis, Ashutosh Chauhan)
Signed-off-by: Laszlo Bodor <bo...@gmail.com>
---
.../org/apache/tez/dag/api/TezConfiguration.java | 9 +++++
.../tez/dag/app/launcher/TezLocalCacheManager.java | 10 ++++--
.../dag/app/launcher/TestTezLocalCacheManager.java | 41 ++++++++++++++++++++++
3 files changed, 58 insertions(+), 2 deletions(-)
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 8410f60..ed59af8 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -1725,6 +1725,15 @@ public class TezConfiguration extends Configuration {
public static final boolean TEZ_LOCAL_MODE_DEFAULT = false;
/**
+ * String value. TezLocalCacheManager uses this folder as a root for temp and localized files.
+ */
+ @ConfigurationScope(Scope.VERTEX)
+ @ConfigurationProperty
+ public static final String TEZ_LOCAL_CACHE_ROOT_FOLDER = TEZ_PREFIX + "local.cache.root.folder";
+
+ public static final String TEZ_LOCAL_CACHE_ROOT_FOLDER_DEFAULT = ".";
+
+ /**
* Tez AM Inline Mode flag. Not valid till Tez-684 get checked-in
*/
@Private
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/TezLocalCacheManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/TezLocalCacheManager.java
index 9bcbb15..f4892ab 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/TezLocalCacheManager.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/TezLocalCacheManager.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.util.FSDownload;
+import org.apache.tez.dag.api.TezConfiguration;
import java.io.IOException;
import java.nio.file.Files;
@@ -63,7 +64,7 @@ public class TezLocalCacheManager {
this.fileContext = FileContext.getLocalFSFileContext();
this.resources = resources;
this.conf = conf;
- this.tempDir = Files.createTempDirectory(Paths.get("."), "tez-local-cache");
+ this.tempDir = Files.createTempDirectory(getLocalCacheRoot(), "tez-local-cache");
}
/**
@@ -72,7 +73,7 @@ public class TezLocalCacheManager {
* @throws IOException when an error occurs in download or link
*/
public void localize() throws IOException {
- String absPath = Paths.get(".").toAbsolutePath().normalize().toString();
+ String absPath = getLocalCacheRoot().toAbsolutePath().normalize().toString();
Path cwd = fileContext.makeQualified(new Path(absPath));
ExecutorService threadPool = null;
@@ -181,6 +182,11 @@ public class TezLocalCacheManager {
}
}
+ private java.nio.file.Path getLocalCacheRoot() {
+ return Paths.get(conf.get(TezConfiguration.TEZ_LOCAL_CACHE_ROOT_FOLDER,
+ TezConfiguration.TEZ_LOCAL_CACHE_ROOT_FOLDER_DEFAULT));
+ }
+
/**
* Wrapper to keep track of download path and link path.
*/
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestTezLocalCacheManager.java b/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestTezLocalCacheManager.java
index beca047..5596dc8 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestTezLocalCacheManager.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestTezLocalCacheManager.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.tez.dag.api.TezConfiguration;
import org.junit.Assert;
import org.junit.Test;
@@ -110,4 +111,44 @@ public class TestTezLocalCacheManager {
ret.setTimestamp(fs.getFileStatus(p).getModificationTime());
return ret;
}
+
+ @Test
+ public void testLocalizeRootDirectory() throws URISyntaxException, IOException {
+ // default directory
+ Map<String, LocalResource> resources = new HashMap<>();
+
+ LocalResource resourceOne = createFile("content-one");
+ resources.put("file-one", resourceOne);
+
+ TezLocalCacheManager manager = new TezLocalCacheManager(resources, new Configuration());
+
+ try {
+ Assert.assertFalse(Files.exists(Paths.get("./file-one")));
+ manager.localize();
+ Assert.assertTrue(Files.exists(Paths.get("./file-one")));
+
+ } finally {
+ manager.cleanup();
+ Assert.assertFalse(Files.exists(Paths.get("./file-one")));
+ }
+
+ // configured directory
+ Configuration conf = new Configuration();
+ conf.set(TezConfiguration.TEZ_LOCAL_CACHE_ROOT_FOLDER, "target");
+ manager = new TezLocalCacheManager(resources, conf);
+
+ try {
+ // files don't exist at all
+ Assert.assertFalse(Files.exists(Paths.get("./file-one")));
+ Assert.assertFalse(Files.exists(Paths.get("./target/file-one")));
+ manager.localize();
+ // file appears only at configured location
+ Assert.assertFalse(Files.exists(Paths.get("./file-one")));
+ Assert.assertTrue(Files.exists(Paths.get("./target/file-one")));
+
+ } finally {
+ manager.cleanup();
+ Assert.assertFalse(Files.exists(Paths.get("./target/file-one")));
+ }
+ }
}